gistfile1.txt

W0410 09:17:10.471299       1 client_config.go:529] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I0410 09:17:10.474541       1 options.go:145] Creating event broadcaster
I0410 09:17:10.474943       1 reflector.go:202] Starting reflector *v1alpha1.Pilot (30s) from github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73
I0410 09:17:10.474977       1 reflector.go:240] Listing and watching *v1alpha1.Pilot from github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73
I0410 09:17:10.475046       1 reflector.go:202] Starting reflector *v1alpha1.ElasticsearchCluster (30s) from github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73
I0410 09:17:10.475077       1 reflector.go:240] Listing and watching *v1alpha1.ElasticsearchCluster from github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73
I0410 09:17:10.574885       1 shared_informer.go:122] caches populated
I0410 09:17:10.574941       1 genericpilot.go:43] Starting generic pilot controller
I0410 09:17:10.575101       1 leaderelection.go:174] attempting to acquire leader lease...
I0410 09:17:10.575714       1 genericpilot.go:104] Starting generic pilot controller
I0410 09:17:10.581182       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:10.581245       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:10.675799       1 shared_informer.go:122] caches populated
I0410 09:17:10.675854       1 controller.go:26] start worker loop
I0410 09:17:10.675916       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (29.801µs)
I0410 09:17:10.675937       1 controller.go:28] processed work item
I0410 09:17:10.675958       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (9.7µs)
I0410 09:17:10.675970       1 controller.go:28] processed work item
I0410 09:17:10.675991       1 hook.go:118] Executing PreStart hook 'WriteConfig'
I0410 09:17:10.676290       1 write_config.go:31] Considering file "..2018_04_10_09_16_30.732468600" (path: "/etc/pilot/elasticsearch/config/..2018_04_10_09_16_30.732468600") when writing elasticsearch config
I0410 09:17:10.676384       1 write_config.go:31] Considering file "..data" (path: "/etc/pilot/elasticsearch/config/..2018_04_10_09_16_30.732468600") when writing elasticsearch config
I0410 09:17:10.676550       1 write_config.go:31] Considering file "elasticsearch.yml" (path: "/etc/pilot/elasticsearch/config/..2018_04_10_09_16_30.732468600/elasticsearch.yml") when writing elasticsearch config
I0410 09:17:10.676572       1 write_config.go:41] Copying config file from "/etc/pilot/elasticsearch/config/..2018_04_10_09_16_30.732468600/elasticsearch.yml" to "/usr/share/elasticsearch/config/elasticsearch.yml"
I0410 09:17:10.696244       1 hook.go:123] Executed PreStart hook "WriteConfig"
I0410 09:17:10.696280       1 hook.go:118] Executing PreStart hook 'InstallPlugins'
I0410 09:17:11.653690       1 install_plugin.go:23] There are 13 plugins already installed: map[ingest-user-agent:{} x-pack:{} 	x-pack-security:{} :{} 	x-pack-monitoring:{} 	x-pack-upgrade:{} ingest-geoip:{} 	x-pack-core:{} 	x-pack-deprecation:{} 	x-pack-graph:{} 	x-pack-logstash:{} 	x-pack-ml:{} 	x-pack-watcher:{}]
I0410 09:17:11.653800       1 hook.go:123] Executed PreStart hook "InstallPlugins"
I0410 09:17:11.654418       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:11.655556       1 event.go:218] Event(v1.ObjectReference{Kind:"Pilot", Namespace:"logging", Name:"es-logging-mixed-3", UID:"bd1a23ab-37e8-11e8-bd07-56967cf91fd8", APIVersion:"navigator.jetstack.io", ResourceVersion:"264998", FieldPath:""}): type: 'Normal' reason: 'ProcessStarted' Subprocess "76" started
I0410 09:17:11.725632       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (1.049643613s)
I0410 09:17:11.725673       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:11.725706       1 controller.go:28] processed work item
I0410 09:17:11.725737       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (19.001µs)
I0410 09:17:11.725751       1 controller.go:28] processed work item
I0410 09:17:11.725769       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (8.1µs)
I0410 09:17:11.725781       1 controller.go:28] processed work item
I0410 09:17:11.725797       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (7µs)
I0410 09:17:11.725818       1 controller.go:28] processed work item
I0410 09:17:11.725836       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (8.7µs)
I0410 09:17:11.725848       1 controller.go:28] processed work item
I0410 09:17:11.725880       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.901µs)
I0410 09:17:11.725902       1 controller.go:28] processed work item
I0410 09:17:11.730864       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:11.730895       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:11.730910       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:11.751851       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (21.002752ms)
I0410 09:17:11.751880       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: [local elasticsearch client not available, Operation cannot be fulfilled on pilots.navigator.jetstack.io "es-logging-mixed-3": the object has been modified; please apply your changes to the latest version and try again]
I0410 09:17:11.751924       1 controller.go:28] processed work item
I0410 09:17:11.751966       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:11.751988       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:11.752002       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:11.756512       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (4.554898ms)
I0410 09:17:11.756554       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:11.756603       1 controller.go:28] processed work item
I0410 09:17:11.762094       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:11.762130       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:11.762145       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:11.765882       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (3.801982ms)
I0410 09:17:11.765912       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:11.765955       1 controller.go:28] processed work item
I0410 09:17:11.806121       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:11.806152       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:11.806166       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:11.812586       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (6.49474ms)
I0410 09:17:11.812618       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:11.812654       1 controller.go:28] processed work item
I0410 09:17:11.892852       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:11.892891       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:11.892907       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:11.897414       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (4.610399ms)
I0410 09:17:11.897445       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:11.897496       1 controller.go:28] processed work item
I0410 09:17:12.057689       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:12.057728       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:12.057767       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:12.072027       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (14.36471ms)
I0410 09:17:12.072067       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:12.072109       1 controller.go:28] processed work item
I0410 09:17:12.392317       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:12.392357       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:12.392373       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:12.396657       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (4.369494ms)
I0410 09:17:12.396688       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:12.396832       1 controller.go:28] processed work item
I0410 09:17:13.036945       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:13.036983       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:13.037010       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:13.048319       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (11.402846ms)
I0410 09:17:13.048351       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:13.048388       1 controller.go:28] processed work item
[2018-04-10T09:17:13,456][INFO ][o.e.n.Node               ] [es-logging-mixed-3] initializing ...
[2018-04-10T09:17:13,569][INFO ][o.e.e.NodeEnvironment    ] [es-logging-mixed-3] using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/pxd/pxd846852866481327394)]], net usable_space [178.7gb], net total_space [245.9gb], types [ext4]
[2018-04-10T09:17:13,569][INFO ][o.e.e.NodeEnvironment    ] [es-logging-mixed-3] heap size [989.8mb], compressed ordinary object pointers [true]
[2018-04-10T09:17:13,751][INFO ][o.e.n.Node               ] [es-logging-mixed-3] node name [es-logging-mixed-3], node ID [RJUjUFwxQDix2m5Q6EdrEA]
[2018-04-10T09:17:13,751][INFO ][o.e.n.Node               ] [es-logging-mixed-3] version[6.2.3], pid[76], build[c59ff00/2018-03-13T10:06:29.741383Z], OS[Linux/4.14.32-coreos/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_161/25.161-b14]
[2018-04-10T09:17:13,751][INFO ][o.e.n.Node               ] [es-logging-mixed-3] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch.J0MImCH4, -XX:+HeapDumpOnOutOfMemoryError, -XX:+PrintGCDetails, -XX:+PrintGCDateStamps, -XX:+PrintTenuringDistribution, -XX:+PrintGCApplicationStoppedTime, -Xloggc:logs/gc.log, -XX:+UseGCLogFileRotation, -XX:NumberOfGCLogFiles=32, -XX:GCLogFileSize=64m, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config]
I0410 09:17:14.181004       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:14.181099       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:14.328611       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:14.328653       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:14.328668       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:14.425427       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (96.850587ms)
I0410 09:17:14.425519       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:14.425604       1 controller.go:28] processed work item
E0410 09:17:15.477243       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:17:15,895][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [aggs-matrix-stats]
[2018-04-10T09:17:15,895][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [analysis-common]
[2018-04-10T09:17:15,895][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [ingest-common]
[2018-04-10T09:17:15,895][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [lang-expression]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [lang-mustache]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [lang-painless]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [mapper-extras]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [parent-join]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [percolator]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [rank-eval]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [reindex]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [repository-url]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [transport-netty4]
[2018-04-10T09:17:15,896][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded module [tribe]
[2018-04-10T09:17:15,897][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [ingest-geoip]
[2018-04-10T09:17:15,897][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [ingest-user-agent]
[2018-04-10T09:17:15,897][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-core]
[2018-04-10T09:17:15,897][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-deprecation]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-graph]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-logstash]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-ml]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-monitoring]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-security]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-upgrade]
[2018-04-10T09:17:15,898][INFO ][o.e.p.PluginsService     ] [es-logging-mixed-3] loaded plugin [x-pack-watcher]
I0410 09:17:16.335472       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.101µs)
I0410 09:17:16.335574       1 controller.go:28] processed work item
I0410 09:17:16.985905       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:16.985946       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:16.985964       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:16.990419       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (4.545398ms)
I0410 09:17:16.990476       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:16.990517       1 controller.go:28] processed work item
I0410 09:17:17.235754       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:17.236264       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:17.535370       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.9µs)
I0410 09:17:17.535410       1 controller.go:28] processed work item
I0410 09:17:17.735536       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.601µs)
I0410 09:17:17.735576       1 controller.go:28] processed work item
[2018-04-10T09:17:18,936][INFO ][o.e.x.m.j.p.l.CppLogMessageHandler] [controller/155] [Main.cc@128] controller (64 bit): Version 6.2.3 (Build e43a9a2b267ef4) Copyright (c) 2018 Elasticsearch BV
[2018-04-10T09:17:19,524][INFO ][o.e.d.DiscoveryModule    ] [es-logging-mixed-3] using discovery type [zen]
I0410 09:17:20.259656       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:20.259701       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:20,313][INFO ][o.e.n.Node               ] [es-logging-mixed-3] initialized
[2018-04-10T09:17:20,314][INFO ][o.e.n.Node               ] [es-logging-mixed-3] starting ...
[2018-04-10T09:17:20,462][INFO ][o.e.t.TransportService   ] [es-logging-mixed-3] publish_address {10.2.3.163:9300}, bound_addresses {[::]:9300}
E0410 09:17:20.479613       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:17:20,508][INFO ][o.e.b.BootstrapChecks    ] [es-logging-mixed-3] bound or publishing to a non-loopback address, enforcing bootstrap checks
I0410 09:17:20.681261       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23µs)
I0410 09:17:20.681303       1 controller.go:28] processed work item
I0410 09:17:21.741606       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.6µs)
I0410 09:17:21.741650       1 controller.go:28] processed work item
I0410 09:17:21.741672       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10µs)
I0410 09:17:21.741686       1 controller.go:28] processed work item
I0410 09:17:21.741704       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (8.3µs)
I0410 09:17:21.741769       1 controller.go:28] processed work item
I0410 09:17:22.110731       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:22.110791       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:22.110819       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:22.118174       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (7.41986ms)
I0410 09:17:22.118202       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:22.118312       1 controller.go:28] processed work item
I0410 09:17:23.913416       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:23.913446       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:17:25.484913       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:17:25,528][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:17:26.075156       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:26.075185       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:26.340783       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.3µs)
I0410 09:17:26.340872       1 controller.go:28] processed work item
I0410 09:17:27.135462       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.401µs)
I0410 09:17:27.135521       1 controller.go:28] processed work item
I0410 09:17:27.540735       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27.9µs)
I0410 09:17:27.540783       1 controller.go:28] processed work item
I0410 09:17:27.740909       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (44.701µs)
I0410 09:17:27.740950       1 controller.go:28] processed work item
I0410 09:17:28.135614       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.301µs)
I0410 09:17:28.135659       1 controller.go:28] processed work item
I0410 09:17:28.455290       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:28.455321       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:28,532][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:17:30.489663       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:17:30.686760       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.5µs)
I0410 09:17:30.686805       1 controller.go:28] processed work item
I0410 09:17:30.692406       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:30.692436       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:31.746972       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (28.801µs)
I0410 09:17:31.747013       1 controller.go:28] processed work item
I0410 09:17:31.747036       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.9µs)
I0410 09:17:31.747048       1 controller.go:28] processed work item
I0410 09:17:31.747066       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.3µs)
I0410 09:17:31.747085       1 controller.go:28] processed work item
I0410 09:17:32.358492       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:17:32.358529       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:17:32.358544       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:17:32.370862       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (12.415564ms)
I0410 09:17:32.370890       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:17:32.370948       1 controller.go:28] processed work item
I0410 09:17:33.419902       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:33.419935       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:33,533][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:17:35.492300       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:17:36,534][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:17:36.660767       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:36.660798       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:37.140980       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (31.8µs)
I0410 09:17:37.141023       1 controller.go:28] processed work item
I0410 09:17:37.546175       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.3µs)
I0410 09:17:37.546242       1 controller.go:28] processed work item
I0410 09:17:37.735520       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18µs)
I0410 09:17:37.735559       1 controller.go:28] processed work item
I0410 09:17:38.141073       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.001µs)
I0410 09:17:38.141117       1 controller.go:28] processed work item
I0410 09:17:38.335493       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.6µs)
I0410 09:17:38.335533       1 controller.go:28] processed work item
I0410 09:17:38.535629       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.7µs)
I0410 09:17:38.535680       1 controller.go:28] processed work item
E0410 09:17:40.494892       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:17:40.531258       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
[2018-04-10T09:17:40,546][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:17:40.618659       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:40.618687       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:40.692090       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (32.401µs)
I0410 09:17:40.692163       1 controller.go:28] processed work item
I0410 09:17:41.752470       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.3µs)
I0410 09:17:41.752512       1 controller.go:28] processed work item
I0410 09:17:41.752538       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (14.6µs)
I0410 09:17:41.752552       1 controller.go:28] processed work item
I0410 09:17:43.137876       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:43.137914       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:43,551][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:17:43,552][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:17:45.497304       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:17:46.056634       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:46.056667       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:46,553][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:17:46,554][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:17:47.551542       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.6µs)
I0410 09:17:47.551583       1 controller.go:28] processed work item
I0410 09:17:47.740983       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (40.501µs)
I0410 09:17:47.741024       1 controller.go:28] processed work item
I0410 09:17:48.041660       1 logs.go:41] http: panic serving 10.0.16.9:41090: runtime error: invalid memory address or nil pointer dereference
goroutine 256 [running]:
net/http.(*conn).serve.func1(0xc4200a15e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3c80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021eba0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420040b40, 0x1b57140, 0xc4204e3c80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420144800, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc4204b6a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc4204b6a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc4204b6a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4200a15e0, 0x1b570c0, 0xc42041b540)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:17:48.135446       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.9µs)
I0410 09:17:48.135497       1 controller.go:28] processed work item
I0410 09:17:48.335645       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (35.501µs)
I0410 09:17:48.335686       1 controller.go:28] processed work item
I0410 09:17:48.341006       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (15.001µs)
I0410 09:17:48.341060       1 controller.go:28] processed work item
I0410 09:17:48.541027       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.801µs)
I0410 09:17:48.541069       1 controller.go:28] processed work item
I0410 09:17:48.735444       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (11.8µs)
I0410 09:17:48.735484       1 controller.go:28] processed work item
I0410 09:17:48.824197       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:48.824267       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:17:49.135494       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.401µs)
I0410 09:17:49.135545       1 controller.go:28] processed work item
[2018-04-10T09:17:49,555][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:17:49,555][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:17:50.499862       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:17:50,530][WARN ][o.e.n.Node               ] [es-logging-mixed-3] timed out while waiting for initial discovery state - timeout: 30s
[2018-04-10T09:17:50,541][INFO ][o.e.h.n.Netty4HttpServerTransport] [es-logging-mixed-3] publish_address {10.2.3.163:9200}, bound_addresses {[::]:9200}
[2018-04-10T09:17:50,541][INFO ][o.e.n.Node               ] [es-logging-mixed-3] started
I0410 09:17:50.697475       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (59.402µs)
I0410 09:17:50.697513       1 controller.go:28] processed work item
I0410 09:17:51.757842       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (28.7µs)
I0410 09:17:51.757887       1 controller.go:28] processed work item
I0410 09:17:51.757936       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.901µs)
I0410 09:17:51.757949       1 controller.go:28] processed work item
I0410 09:17:51.954320       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:51.954346       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:52,556][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:17:54.639314       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:54.639346       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:17:55.621908       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:17:57.347441       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:17:57.347484       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:17:57,557][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:17:58.040712       1 logs.go:41] http: panic serving 10.0.16.9:41250: runtime error: invalid memory address or nil pointer dereference
goroutine 281 [running]:
net/http.(*conn).serve.func1(0xc42055c0a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420456a80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4201e3320, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420040b40, 0x1b57140, 0xc420456a80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c000, 0xc4204b7200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c000, 0xc4204b7200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c000, 0xc4204b7200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42055c0a0, 0x1b570c0, 0xc42041bc00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:17:58.140982       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (51.702µs)
I0410 09:17:58.141052       1 controller.go:28] processed work item
I0410 09:17:58.341084       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.9µs)
I0410 09:17:58.341127       1 controller.go:28] processed work item
I0410 09:17:58.735525       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (37.901µs)
I0410 09:17:58.735571       1 controller.go:28] processed work item
I0410 09:17:58.740792       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (10.4µs)
I0410 09:17:58.740827       1 controller.go:28] processed work item
I0410 09:17:59.135575       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (31.201µs)
I0410 09:17:59.135621       1 controller.go:28] processed work item
I0410 09:17:59.140746       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.7µs)
I0410 09:17:59.140780       1 controller.go:28] processed work item
I0410 09:17:59.335482       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (15.5µs)
I0410 09:17:59.335530       1 controller.go:28] processed work item
I0410 09:17:59.735491       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17µs)
I0410 09:17:59.735530       1 controller.go:28] processed work item
[2018-04-10T09:18:00,558][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:18:00.631781       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:00.702877       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (70.902µs)
I0410 09:18:00.702928       1 controller.go:28] processed work item
I0410 09:18:00.982892       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:00.982924       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:01.763309       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.8µs)
I0410 09:18:01.763359       1 controller.go:28] processed work item
I0410 09:18:01.763401       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (18.4µs)
I0410 09:18:01.763414       1 controller.go:28] processed work item
I0410 09:18:03.511970       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:03.512002       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:18:05,558][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:18:05.644921       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:06.013085       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:06.013117       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:08.040734       1 logs.go:41] http: panic serving 10.0.16.9:41408: runtime error: invalid memory address or nil pointer dereference
goroutine 317 [running]:
net/http.(*conn).serve.func1(0xc4206c86e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420457560, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420205260, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42054eb40, 0x1b57140, 0xc420457560, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc420567800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc420567800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc420567800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c86e0, 0x1b570c0, 0xc42060e980)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:18:08,559][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:18:08.740939       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (43.501µs)
I0410 09:18:08.740985       1 controller.go:28] processed work item
I0410 09:18:08.883412       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:08.883434       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:09.140948       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (32.601µs)
I0410 09:18:09.140989       1 controller.go:28] processed work item
I0410 09:18:09.340825       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.5µs)
I0410 09:18:09.340883       1 controller.go:28] processed work item
I0410 09:18:09.535663       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (14.701µs)
I0410 09:18:09.535700       1 controller.go:28] processed work item
I0410 09:18:09.535744       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (31.701µs)
I0410 09:18:09.535768       1 controller.go:28] processed work item
I0410 09:18:09.740794       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.001µs)
I0410 09:18:09.740873       1 controller.go:28] processed work item
I0410 09:18:10.135541       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.801µs)
I0410 09:18:10.135581       1 controller.go:28] processed work item
I0410 09:18:10.335509       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.501µs)
I0410 09:18:10.335548       1 controller.go:28] processed work item
I0410 09:18:10.531508       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
E0410 09:18:10.653696       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:10.708260       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (20.9µs)
I0410 09:18:10.708314       1 controller.go:28] processed work item
I0410 09:18:11.768685       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (32.601µs)
I0410 09:18:11.768730       1 controller.go:28] processed work item
I0410 09:18:11.768751       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.6µs)
I0410 09:18:11.768764       1 controller.go:28] processed work item
I0410 09:18:12.257583       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:12.257645       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:18:12,574][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:18:13.331168       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:18:13.331242       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:18:13.331259       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:18:13.424406       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (93.271091ms)
I0410 09:18:13.424436       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:18:13.424469       1 controller.go:28] processed work item
[2018-04-10T09:18:15,576][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:18:15,576][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:18:15.664565       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:16.332653       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:16.332687       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:18.040600       1 logs.go:41] http: panic serving 10.0.16.9:41598: runtime error: invalid memory address or nil pointer dereference
goroutine 331 [running]:
net/http.(*conn).serve.func1(0xc42055c640)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420611320, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bba70, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42061ab40, 0x1b57140, 0xc420611320, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5180, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8460, 0xc4205a4900)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8460, 0xc4205a4900)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8460, 0xc4205a4900)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42055c640, 0x1b570c0, 0xc420624ac0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:18:18,578][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:18:18,578][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:18:19.045667       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:19.045708       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:19.540994       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.201µs)
I0410 09:18:19.541039       1 controller.go:28] processed work item
I0410 09:18:19.541061       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (10.2µs)
I0410 09:18:19.541075       1 controller.go:28] processed work item
I0410 09:18:19.935642       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.401µs)
I0410 09:18:19.935689       1 controller.go:28] processed work item
I0410 09:18:20.141070       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (47.401µs)
I0410 09:18:20.141118       1 controller.go:28] processed work item
I0410 09:18:20.340990       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (41.201µs)
I0410 09:18:20.341047       1 controller.go:28] processed work item
E0410 09:18:20.677036       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:20.713668       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (51.101µs)
I0410 09:18:20.713730       1 controller.go:28] processed work item
I0410 09:18:20.735657       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (10.9µs)
I0410 09:18:20.735698       1 controller.go:28] processed work item
[2018-04-10T09:18:21,580][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:18:21,581][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:18:21.763463       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:21.763492       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:21.774278       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (19.4µs)
I0410 09:18:21.774314       1 controller.go:28] processed work item
I0410 09:18:21.774337       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.6µs)
I0410 09:18:21.774351       1 controller.go:28] processed work item
[2018-04-10T09:18:24,582][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:18:25.577578       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:25.577609       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:18:25.689127       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:28.041045       1 logs.go:41] http: panic serving 10.0.16.9:41800: runtime error: invalid memory address or nil pointer dereference
goroutine 372 [running]:
net/http.(*conn).serve.func1(0xc4201fcd20)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420584ba0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4205dc7e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420040b40, 0x1b57140, 0xc420584ba0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420329080, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42070c000, 0xc42058af00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42070c000, 0xc42058af00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42070c000, 0xc42058af00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fcd20, 0x1b570c0, 0xc42058c840)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:18:28.078043       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:28.078135       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:29.546513       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (43.001µs)
I0410 09:18:29.546554       1 controller.go:28] processed work item
[2018-04-10T09:18:29,583][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:18:29.940992       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.3µs)
I0410 09:18:29.941047       1 controller.go:28] processed work item
I0410 09:18:30.346332       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.601µs)
I0410 09:18:30.346380       1 controller.go:28] processed work item
I0410 09:18:30.536540       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.501µs)
I0410 09:18:30.536574       1 controller.go:28] processed work item
E0410 09:18:30.698102       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:30.719057       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (16.701µs)
I0410 09:18:30.719086       1 controller.go:28] processed work item
I0410 09:18:30.741012       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (13.801µs)
I0410 09:18:30.741098       1 controller.go:28] processed work item
I0410 09:18:31.335806       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.7µs)
I0410 09:18:31.336000       1 controller.go:28] processed work item
I0410 09:18:31.779671       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (47.501µs)
I0410 09:18:31.779815       1 controller.go:28] processed work item
I0410 09:18:31.779886       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21µs)
I0410 09:18:31.779966       1 controller.go:28] processed work item
I0410 09:18:32.159865       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:32.160033       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:18:32,584][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:18:35.707706       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:35.836810       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:35.836844       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:18:37,584][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:18:38.040714       1 logs.go:41] http: panic serving 10.0.16.9:41986: runtime error: invalid memory address or nil pointer dereference
goroutine 385 [running]:
net/http.(*conn).serve.func1(0xc4206c8fa0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420708600, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4205c1980, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42061ab40, 0x1b57140, 0xc420708600, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f3680, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c2a0, 0xc4205bf600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c2a0, 0xc4205bf600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c2a0, 0xc4205bf600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c8fa0, 0x1b570c0, 0xc42060f800)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:18:39.098863       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:39.098897       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:39.551982       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (60.501µs)
I0410 09:18:39.552038       1 controller.go:28] processed work item
I0410 09:18:40.351680       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.5µs)
I0410 09:18:40.351723       1 controller.go:28] processed work item
I0410 09:18:40.531981       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:18:40.541887       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.5µs)
I0410 09:18:40.541938       1 controller.go:28] processed work item
[2018-04-10T09:18:40,585][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:18:40.716409       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:40.724458       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (20.401µs)
I0410 09:18:40.724495       1 controller.go:28] processed work item
I0410 09:18:41.135603       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.101µs)
I0410 09:18:41.135671       1 controller.go:28] processed work item
I0410 09:18:41.178780       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:41.178810       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:41.336154       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.9µs)
I0410 09:18:41.336298       1 controller.go:28] processed work item
I0410 09:18:41.341573       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.901µs)
I0410 09:18:41.341593       1 controller.go:28] processed work item
I0410 09:18:41.535554       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.1µs)
I0410 09:18:41.535590       1 controller.go:28] processed work item
I0410 09:18:41.785080       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (31.2µs)
I0410 09:18:41.785116       1 controller.go:28] processed work item
I0410 09:18:41.785146       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (18.8µs)
I0410 09:18:41.785159       1 controller.go:28] processed work item
I0410 09:18:41.935847       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.5µs)
I0410 09:18:41.935888       1 controller.go:28] processed work item
I0410 09:18:43.614619       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:43.614654       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:18:44,604][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:18:45.725950       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:47.076836       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:47.076886       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:18:47,605][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:18:47,606][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:18:48.041021       1 logs.go:41] http: panic serving 10.0.16.9:42206: runtime error: invalid memory address or nil pointer dereference
goroutine 393 [running]:
net/http.(*conn).serve.func1(0xc4206c9220)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420708de0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42076e600, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42061ab40, 0x1b57140, 0xc420708de0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f3d00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c380, 0xc420774000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c380, 0xc420774000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c380, 0xc420774000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c9220, 0x1b570c0, 0xc42060fc40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:18:50,607][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:18:50,607][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:18:50.729796       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.201µs)
I0410 09:18:50.729864       1 controller.go:28] processed work item
E0410 09:18:50.735104       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:18:51.141035       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.4µs)
I0410 09:18:51.141076       1 controller.go:28] processed work item
I0410 09:18:51.341542       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.801µs)
I0410 09:18:51.341587       1 controller.go:28] processed work item
I0410 09:18:51.422336       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:51.422369       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:51.540855       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.4µs)
I0410 09:18:51.540956       1 controller.go:28] processed work item
I0410 09:18:51.735680       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (41.7µs)
I0410 09:18:51.735746       1 controller.go:28] processed work item
I0410 09:18:51.735779       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.201µs)
I0410 09:18:51.735803       1 controller.go:28] processed work item
I0410 09:18:51.790442       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (19.701µs)
I0410 09:18:51.790482       1 controller.go:28] processed work item
I0410 09:18:51.790515       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.5µs)
I0410 09:18:51.790538       1 controller.go:28] processed work item
I0410 09:18:51.935482       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (13µs)
I0410 09:18:51.935571       1 controller.go:28] processed work item
I0410 09:18:51.941276       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (14.101µs)
I0410 09:18:51.941311       1 controller.go:28] processed work item
I0410 09:18:52.535664       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.201µs)
I0410 09:18:52.535712       1 controller.go:28] processed work item
[2018-04-10T09:18:53,609][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:18:53,610][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:18:53.618803       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:53.618873       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:18:55.744327       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:18:56,611][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:18:57.053865       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:57.053896       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:18:58.041696       1 logs.go:41] http: panic serving 10.0.16.9:42484: runtime error: invalid memory address or nil pointer dereference
goroutine 271 [running]:
net/http.(*conn).serve.func1(0xc420171ea0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205151a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420339470, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc4205151a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420319080, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c000, 0xc4202f6e00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c000, 0xc4202f6e00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c000, 0xc4202f6e00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420171ea0, 0x1b570c0, 0xc42041a9c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:18:59.200922       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:18:59.200957       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:00.735190       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.4µs)
I0410 09:19:00.735267       1 controller.go:28] processed work item
E0410 09:19:00.752710       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:19:01,611][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:19:01.741059       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (21.601µs)
I0410 09:19:01.741108       1 controller.go:28] processed work item
I0410 09:19:01.741128       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (9.4µs)
I0410 09:19:01.741140       1 controller.go:28] processed work item
I0410 09:19:01.795784       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (20.201µs)
I0410 09:19:01.795822       1 controller.go:28] processed work item
I0410 09:19:01.795844       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.4µs)
I0410 09:19:01.795858       1 controller.go:28] processed work item
I0410 09:19:01.940822       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.501µs)
I0410 09:19:01.940875       1 controller.go:28] processed work item
I0410 09:19:02.335620       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.2µs)
I0410 09:19:02.335674       1 controller.go:28] processed work item
I0410 09:19:02.541151       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.1µs)
I0410 09:19:02.541204       1 controller.go:28] processed work item
I0410 09:19:02.871468       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:02.871504       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:03.135767       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.201µs)
I0410 09:19:03.135802       1 controller.go:28] processed work item
[2018-04-10T09:19:04,612][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:19:05.600070       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:05.600105       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:19:05.761881       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:19:08.023543       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:08.023570       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:08.040601       1 logs.go:41] http: panic serving 10.0.16.9:42702: runtime error: invalid memory address or nil pointer dereference
goroutine 441 [running]:
net/http.(*conn).serve.func1(0xc420436280)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420515f20, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420290660, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205fdb40, 0x1b57140, 0xc420515f20, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420319e00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc4202f7800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc4202f7800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc4202f7800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436280, 0x1b570c0, 0xc42041b4c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:19:09,613][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:19:10.532285       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:19:10.740617       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.901µs)
I0410 09:19:10.740660       1 controller.go:28] processed work item
E0410 09:19:10.770850       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:19:11.326839       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:11.326888       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:11.746842       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.401µs)
I0410 09:19:11.746883       1 controller.go:28] processed work item
I0410 09:19:11.801147       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (14.8µs)
I0410 09:19:11.801179       1 controller.go:28] processed work item
I0410 09:19:11.801200       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.2µs)
I0410 09:19:11.801232       1 controller.go:28] processed work item
I0410 09:19:11.946292       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (32.9µs)
I0410 09:19:11.946338       1 controller.go:28] processed work item
I0410 09:19:12.341113       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (26.5µs)
I0410 09:19:12.341157       1 controller.go:28] processed work item
I0410 09:19:12.535615       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.2µs)
I0410 09:19:12.535703       1 controller.go:28] processed work item
[2018-04-10T09:19:12,614][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:19:12.935981       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (33.801µs)
I0410 09:19:12.936024       1 controller.go:28] processed work item
I0410 09:19:12.936127       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.901µs)
I0410 09:19:12.936155       1 controller.go:28] processed work item
I0410 09:19:13.141131       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (35.301µs)
I0410 09:19:13.141172       1 controller.go:28] processed work item
I0410 09:19:13.735654       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (14.601µs)
I0410 09:19:13.735766       1 controller.go:28] processed work item
I0410 09:19:14.640967       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:14.640998       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:19:15.779230       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:19:16,631][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:17.313686       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:17.313719       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:18.040897       1 logs.go:41] http: panic serving 10.0.16.9:42866: runtime error: invalid memory address or nil pointer dereference
goroutine 481 [running]:
net/http.(*conn).serve.func1(0xc4204365a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e29c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420291560, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc4204e29c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4203e8f00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c2a0, 0xc4202a4100)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c2a0, 0xc4202a4100)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c2a0, 0xc4202a4100)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204365a0, 0x1b570c0, 0xc42041b940)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:19:19,633][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:19:19,634][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:20.334150       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:20.334179       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:20.746005       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.8µs)
I0410 09:19:20.746050       1 controller.go:28] processed work item
E0410 09:19:20.788498       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:19:21.806527       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (20.501µs)
I0410 09:19:21.806564       1 controller.go:28] processed work item
I0410 09:19:21.806596       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (21.7µs)
I0410 09:19:21.806609       1 controller.go:28] processed work item
I0410 09:19:22.540997       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.5µs)
I0410 09:19:22.541042       1 controller.go:28] processed work item
[2018-04-10T09:19:22,636][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:19:22,636][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:22.941499       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.201µs)
I0410 09:19:22.941539       1 controller.go:28] processed work item
I0410 09:19:22.941575       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.201µs)
I0410 09:19:22.941598       1 controller.go:28] processed work item
I0410 09:19:23.335561       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.8µs)
I0410 09:19:23.335598       1 controller.go:28] processed work item
I0410 09:19:23.612338       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:23.612431       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:23.741118       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (32.601µs)
I0410 09:19:23.741178       1 controller.go:28] processed work item
I0410 09:19:24.135531       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.2µs)
I0410 09:19:24.135584       1 controller.go:28] processed work item
[2018-04-10T09:19:25,637][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:19:25,638][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:19:25.796960       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:19:26.225964       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:26.226022       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:28.040889       1 logs.go:41] http: panic serving 10.0.16.9:43028: runtime error: invalid memory address or nil pointer dereference
goroutine 476 [running]:
net/http.(*conn).serve.func1(0xc4206c8960)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426600, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420259530, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc420426600, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5a80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc42075a700)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc42075a700)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc42075a700)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c8960, 0x1b570c0, 0xc420261200)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:19:28,639][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:19:28.907413       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:28.907468       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:30.751382       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.6µs)
I0410 09:19:30.751425       1 controller.go:28] processed work item
E0410 09:19:30.805486       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:19:31.811924       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (20.7µs)
I0410 09:19:31.812012       1 controller.go:28] processed work item
I0410 09:19:31.812053       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (12.3µs)
I0410 09:19:31.812106       1 controller.go:28] processed work item
I0410 09:19:32.546464       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (36.301µs)
I0410 09:19:32.546514       1 controller.go:28] processed work item
I0410 09:19:32.804072       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:32.804113       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:32.946889       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.301µs)
I0410 09:19:32.946921       1 controller.go:28] processed work item
I0410 09:19:33.341029       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (33.201µs)
I0410 09:19:33.341068       1 controller.go:28] processed work item
[2018-04-10T09:19:33,640][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:19:33.735762       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (29.901µs)
I0410 09:19:33.735795       1 controller.go:28] processed work item
I0410 09:19:33.735814       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (9µs)
I0410 09:19:33.735838       1 controller.go:28] processed work item
I0410 09:19:33.935788       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (29.1µs)
I0410 09:19:33.935827       1 controller.go:28] processed work item
I0410 09:19:33.935849       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.301µs)
I0410 09:19:33.935863       1 controller.go:28] processed work item
I0410 09:19:34.138348       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (15.401µs)
I0410 09:19:34.138388       1 controller.go:28] processed work item
I0410 09:19:34.140876       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.3µs)
I0410 09:19:34.140909       1 controller.go:28] processed work item
I0410 09:19:34.535577       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.3µs)
I0410 09:19:34.535620       1 controller.go:28] processed work item
I0410 09:19:34.735558       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (59.001µs)
I0410 09:19:34.735593       1 controller.go:28] processed work item
I0410 09:19:35.677026       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:35.677058       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:19:35.814505       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:19:36,641][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:19:38.040915       1 logs.go:41] http: panic serving 10.0.16.9:43206: runtime error: invalid memory address or nil pointer dereference
goroutine 532 [running]:
net/http.(*conn).serve.func1(0xc4206c8dc0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426c00, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42016cc90, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc420426c00, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420320a00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8460, 0xc42075b800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8460, 0xc42075b800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8460, 0xc42075b800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c8dc0, 0x1b570c0, 0xc4202619c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:19:39.796113       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:39.796168       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:40.532576       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:19:40.756818       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.601µs)
I0410 09:19:40.756863       1 controller.go:28] processed work item
E0410 09:19:40.822643       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:19:41,641][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:19:41.817338       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.001µs)
I0410 09:19:41.817376       1 controller.go:28] processed work item
I0410 09:19:41.817395       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (8.6µs)
I0410 09:19:41.817423       1 controller.go:28] processed work item
I0410 09:19:42.514313       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:42.514357       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:43.741148       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.301µs)
I0410 09:19:43.741188       1 controller.go:28] processed work item
I0410 09:19:43.941235       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.701µs)
I0410 09:19:43.941273       1 controller.go:28] processed work item
I0410 09:19:44.135485       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.9µs)
I0410 09:19:44.135526       1 controller.go:28] processed work item
I0410 09:19:44.143621       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (12.101µs)
I0410 09:19:44.143663       1 controller.go:28] processed work item
I0410 09:19:44.335708       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.201µs)
I0410 09:19:44.335738       1 controller.go:28] processed work item
[2018-04-10T09:19:44,642][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:19:44.740903       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.3µs)
I0410 09:19:44.740939       1 controller.go:28] processed work item
I0410 09:19:45.135723       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.4µs)
I0410 09:19:45.135762       1 controller.go:28] processed work item
E0410 09:19:45.830980       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:19:46.665895       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:46.665925       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:48.040622       1 logs.go:41] http: panic serving 10.0.16.9:43370: runtime error: invalid memory address or nil pointer dereference
goroutine 538 [running]:
net/http.(*conn).serve.func1(0xc4206c9040)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420427080, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42016d860, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc420427080, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420321500, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8540, 0xc42075bf00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8540, 0xc42075bf00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8540, 0xc42075bf00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c9040, 0x1b570c0, 0xc4206240c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:19:48,655][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:48.906397       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:48.906440       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:50.762253       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (58.501µs)
I0410 09:19:50.762321       1 controller.go:28] processed work item
E0410 09:19:50.839361       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:19:51,656][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:19:51,657][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:51.822832       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (33.3µs)
I0410 09:19:51.822900       1 controller.go:28] processed work item
I0410 09:19:51.822947       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.4µs)
I0410 09:19:51.823092       1 controller.go:28] processed work item
I0410 09:19:53.257665       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:53.257718       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:54.141003       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.1µs)
I0410 09:19:54.141065       1 controller.go:28] processed work item
I0410 09:19:54.149017       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.501µs)
I0410 09:19:54.149052       1 controller.go:28] processed work item
I0410 09:19:54.335642       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.901µs)
I0410 09:19:54.335688       1 controller.go:28] processed work item
I0410 09:19:54.341147       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11.901µs)
I0410 09:19:54.341176       1 controller.go:28] processed work item
I0410 09:19:54.535763       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.8µs)
I0410 09:19:54.535802       1 controller.go:28] processed work item
I0410 09:19:54.535821       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (9.4µs)
I0410 09:19:54.535844       1 controller.go:28] processed work item
[2018-04-10T09:19:54,658][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:19:54,658][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:54.735510       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.801µs)
I0410 09:19:54.735556       1 controller.go:28] processed work item
I0410 09:19:55.141187       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (35.5µs)
I0410 09:19:55.141266       1 controller.go:28] processed work item
I0410 09:19:55.441205       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:55.441306       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:55.735678       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (16.5µs)
I0410 09:19:55.735755       1 controller.go:28] processed work item
E0410 09:19:55.847511       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:19:57,660][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:19:57,660][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:19:57.979279       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:19:57.979340       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:19:58.040915       1 logs.go:41] http: panic serving 10.0.16.9:43534: runtime error: invalid memory address or nil pointer dereference
goroutine 552 [running]:
net/http.(*conn).serve.func1(0xc4201fd360)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204576e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bb590, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205eeb40, 0x1b57140, 0xc4204576e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420230a00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42024e000, 0xc420248c00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42024e000, 0xc420248c00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42024e000, 0xc420248c00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd360, 0x1b570c0, 0xc42058ce00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:20:00,662][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:20:00.767650       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.801µs)
I0410 09:20:00.767697       1 controller.go:28] processed work item
E0410 09:20:00.858099       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:01.618654       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:01.618687       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:01.828252       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (39.601µs)
I0410 09:20:01.828305       1 controller.go:28] processed work item
I0410 09:20:01.828327       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.5µs)
I0410 09:20:01.828340       1 controller.go:28] processed work item
I0410 09:20:04.209101       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:04.209155       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:04.341165       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.001µs)
I0410 09:20:04.341234       1 controller.go:28] processed work item
I0410 09:20:04.541226       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.9µs)
I0410 09:20:04.541278       1 controller.go:28] processed work item
I0410 09:20:04.741041       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (38.701µs)
I0410 09:20:04.741094       1 controller.go:28] processed work item
I0410 09:20:04.935619       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.7µs)
I0410 09:20:04.935670       1 controller.go:28] processed work item
I0410 09:20:05.135581       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.1µs)
I0410 09:20:05.135661       1 controller.go:28] processed work item
I0410 09:20:05.135686       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (11.601µs)
I0410 09:20:05.135700       1 controller.go:28] processed work item
I0410 09:20:05.335544       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.901µs)
I0410 09:20:05.335589       1 controller.go:28] processed work item
[2018-04-10T09:20:05,662][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:20:05.741034       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.801µs)
I0410 09:20:05.741077       1 controller.go:28] processed work item
E0410 09:20:05.871392       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:06.137361       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.901µs)
I0410 09:20:06.137415       1 controller.go:28] processed work item
I0410 09:20:06.962376       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:06.962418       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:08.040721       1 logs.go:41] http: panic serving 10.0.16.9:43700: runtime error: invalid memory address or nil pointer dereference
goroutine 596 [running]:
net/http.(*conn).serve.func1(0xc4201fd720)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4207082a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42076ed20, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205eeb40, 0x1b57140, 0xc4207082a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420231b00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42024e0e0, 0xc420249800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42024e0e0, 0xc420249800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42024e0e0, 0xc420249800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd720, 0x1b570c0, 0xc42058dcc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:20:08,663][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:20:10.532898       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:20:10.773002       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.5µs)
I0410 09:20:10.773070       1 controller.go:28] processed work item
E0410 09:20:10.879756       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:11.205834       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:11.205869       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:11.833683       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (36.5µs)
I0410 09:20:11.833770       1 controller.go:28] processed work item
I0410 09:20:11.833818       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.2µs)
I0410 09:20:11.833855       1 controller.go:28] processed work item
[2018-04-10T09:20:13,664][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:20:15.043011       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:15.043063       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:15.140972       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.501µs)
I0410 09:20:15.141023       1 controller.go:28] processed work item
I0410 09:20:15.141064       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.9µs)
I0410 09:20:15.141078       1 controller.go:28] processed work item
I0410 09:20:15.340906       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.001µs)
I0410 09:20:15.340950       1 controller.go:28] processed work item
I0410 09:20:15.536027       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.701µs)
I0410 09:20:15.536075       1 controller.go:28] processed work item
E0410 09:20:15.887910       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:15.937710       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.1µs)
I0410 09:20:15.937756       1 controller.go:28] processed work item
I0410 09:20:15.937798       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.6µs)
I0410 09:20:15.937831       1 controller.go:28] processed work item
I0410 09:20:16.135510       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.801µs)
I0410 09:20:16.135652       1 controller.go:28] processed work item
I0410 09:20:16.142741       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.501µs)
I0410 09:20:16.142834       1 controller.go:28] processed work item
[2018-04-10T09:20:16,664][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:20:16.735603       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.801µs)
I0410 09:20:16.735663       1 controller.go:28] processed work item
I0410 09:20:18.040630       1 logs.go:41] http: panic serving 10.0.16.9:43876: runtime error: invalid memory address or nil pointer dereference
goroutine 604 [running]:
net/http.(*conn).serve.func1(0xc4201fda40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420708ba0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42076fa40, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420708ba0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420144580, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42024e1c0, 0xc4205ba100)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42024e1c0, 0xc4205ba100)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42024e1c0, 0xc4205ba100)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fda40, 0x1b570c0, 0xc4204c6240)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:20:18.971548       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:18.971595       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:20,673][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:20:20.778474       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (27.9µs)
I0410 09:20:20.778522       1 controller.go:28] processed work item
E0410 09:20:20.896096       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:21.839120       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (38.501µs)
I0410 09:20:21.839197       1 controller.go:28] processed work item
I0410 09:20:21.839245       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (33.201µs)
I0410 09:20:21.839270       1 controller.go:28] processed work item
I0410 09:20:22.728542       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:22.728584       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:23,674][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:20:23,675][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:20:25.146337       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.001µs)
I0410 09:20:25.146411       1 controller.go:28] processed work item
I0410 09:20:25.177990       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:25.178016       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:20:25.904448       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:25.943322       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.001µs)
I0410 09:20:25.943371       1 controller.go:28] processed work item
I0410 09:20:26.140925       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (35.101µs)
I0410 09:20:26.140969       1 controller.go:28] processed work item
I0410 09:20:26.535608       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.6µs)
I0410 09:20:26.535656       1 controller.go:28] processed work item
[2018-04-10T09:20:26,676][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:20:26,677][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:20:26.735877       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.3µs)
I0410 09:20:26.735911       1 controller.go:28] processed work item
I0410 09:20:26.735931       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (9.2µs)
I0410 09:20:26.735954       1 controller.go:28] processed work item
I0410 09:20:26.740948       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (13µs)
I0410 09:20:26.741001       1 controller.go:28] processed work item
I0410 09:20:27.335796       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.501µs)
I0410 09:20:27.335846       1 controller.go:28] processed work item
I0410 09:20:28.040887       1 logs.go:41] http: panic serving 10.0.16.9:44032: runtime error: invalid memory address or nil pointer dereference
goroutine 633 [running]:
net/http.(*conn).serve.func1(0xc420437a40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420611920, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075f830, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc420611920, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2580, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c460, 0xc4205aea00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c460, 0xc4205aea00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c460, 0xc4205aea00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437a40, 0x1b570c0, 0xc42049cb40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:20:28.211338       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:28.211398       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:29,678][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:20:29,678][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:20:30.783895       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.701µs)
I0410 09:20:30.783945       1 controller.go:28] processed work item
E0410 09:20:30.912500       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:31.844569       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.9µs)
I0410 09:20:31.844616       1 controller.go:28] processed work item
I0410 09:20:31.844639       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.8µs)
I0410 09:20:31.844652       1 controller.go:28] processed work item
I0410 09:20:32.371305       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:32.371335       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:32,679][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:20:35.151831       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.601µs)
I0410 09:20:35.151878       1 controller.go:28] processed work item
E0410 09:20:35.921153       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:36.014061       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:36.014106       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:36.135606       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.8µs)
I0410 09:20:36.135656       1 controller.go:28] processed work item
I0410 09:20:36.741356       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (44.701µs)
I0410 09:20:36.741425       1 controller.go:28] processed work item
I0410 09:20:36.741478       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (29.201µs)
I0410 09:20:36.741492       1 controller.go:28] processed work item
I0410 09:20:37.336312       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.6µs)
I0410 09:20:37.336382       1 controller.go:28] processed work item
I0410 09:20:37.336426       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.7µs)
I0410 09:20:37.336449       1 controller.go:28] processed work item
I0410 09:20:37.341283       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (41.901µs)
I0410 09:20:37.341333       1 controller.go:28] processed work item
I0410 09:20:37.535456       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.7µs)
I0410 09:20:37.535506       1 controller.go:28] processed work item
[2018-04-10T09:20:37,680][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:20:37.935735       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (14.9µs)
I0410 09:20:37.935784       1 controller.go:28] processed work item
I0410 09:20:38.043603       1 logs.go:41] http: panic serving 10.0.16.9:44202: runtime error: invalid memory address or nil pointer dereference
goroutine 649 [running]:
net/http.(*conn).serve.func1(0xc4206c9f40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420611ec0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42055a540, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc420611ec0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043b780, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c540, 0xc4205af200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c540, 0xc4205af200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c540, 0xc4205af200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c9f40, 0x1b570c0, 0xc42049d140)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:20:40.368632       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:40.368657       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:40.533264       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
[2018-04-10T09:20:40,681][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:20:40.789319       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.8µs)
I0410 09:20:40.789361       1 controller.go:28] processed work item
E0410 09:20:40.930049       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:41.849964       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.401µs)
I0410 09:20:41.850005       1 controller.go:28] processed work item
I0410 09:20:41.850026       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.3µs)
I0410 09:20:41.850040       1 controller.go:28] processed work item
I0410 09:20:44.586158       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:44.586243       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:45,681][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:20:45.939280       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:46.141067       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.101µs)
I0410 09:20:46.141112       1 controller.go:28] processed work item
I0410 09:20:46.808912       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:46.808940       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:47.341679       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.2µs)
I0410 09:20:47.341761       1 controller.go:28] processed work item
I0410 09:20:47.540856       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.7µs)
I0410 09:20:47.540901       1 controller.go:28] processed work item
I0410 09:20:47.735637       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.101µs)
I0410 09:20:47.735687       1 controller.go:28] processed work item
I0410 09:20:47.941038       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.8µs)
I0410 09:20:47.941080       1 controller.go:28] processed work item
I0410 09:20:48.040716       1 logs.go:41] http: panic serving 10.0.16.9:44378: runtime error: invalid memory address or nil pointer dereference
goroutine 620 [running]:
net/http.(*conn).serve.func1(0xc4206c8280)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420456c00, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075ec30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420788b40, 0x1b57140, 0xc420456c00, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420144980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc4205e6700)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc4205e6700)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc4205e6700)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c8280, 0x1b570c0, 0xc42058c600)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:20:48.135538       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.4µs)
I0410 09:20:48.135578       1 controller.go:28] processed work item
I0410 09:20:48.335697       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.7µs)
I0410 09:20:48.335731       1 controller.go:28] processed work item
I0410 09:20:48.335778       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.901µs)
I0410 09:20:48.335789       1 controller.go:28] processed work item
[2018-04-10T09:20:48,682][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:20:49.997450       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:49.997498       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:20:50.794719       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.901µs)
I0410 09:20:50.794785       1 controller.go:28] processed work item
E0410 09:20:50.948054       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:51.855363       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (35.701µs)
I0410 09:20:51.855416       1 controller.go:28] processed work item
I0410 09:20:51.855450       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.4µs)
I0410 09:20:51.855463       1 controller.go:28] processed work item
[2018-04-10T09:20:52,700][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:20:54.228205       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:54.228260       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:55,701][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:20:55,702][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:20:55.956293       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:20:56.146508       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (45.501µs)
I0410 09:20:56.146559       1 controller.go:28] processed work item
I0410 09:20:57.135623       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (29.601µs)
I0410 09:20:57.135672       1 controller.go:28] processed work item
I0410 09:20:57.264737       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:20:57.264806       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:20:57.264834       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:20:57.327690       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (62.96835ms)
I0410 09:20:57.327725       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:20:57.327764       1 controller.go:28] processed work item
I0410 09:20:57.741023       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.401µs)
I0410 09:20:57.741055       1 controller.go:28] processed work item
I0410 09:20:58.040772       1 logs.go:41] http: panic serving 10.0.16.9:44532: runtime error: invalid memory address or nil pointer dereference
goroutine 669 [running]:
net/http.(*conn).serve.func1(0xc4204361e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426d20, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203ba960, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420555b40, 0x1b57140, 0xc420426d20, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dd000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c000, 0xc4202f7400)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c000, 0xc4202f7400)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c000, 0xc4202f7400)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204361e0, 0x1b570c0, 0xc420624900)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:20:58.135514       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.9µs)
I0410 09:20:58.135645       1 controller.go:28] processed work item
I0410 09:20:58.341173       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.501µs)
I0410 09:20:58.341239       1 controller.go:28] processed work item
I0410 09:20:58.341286       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11µs)
I0410 09:20:58.341309       1 controller.go:28] processed work item
I0410 09:20:58.524788       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:20:58.524902       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:20:58,703][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:20:58,704][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:20:58.735683       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.2µs)
I0410 09:20:58.735724       1 controller.go:28] processed work item
I0410 09:20:58.935607       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (28.7µs)
I0410 09:20:58.935642       1 controller.go:28] processed work item
I0410 09:21:00.800116       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.001µs)
I0410 09:21:00.800161       1 controller.go:28] processed work item
E0410 09:21:00.964948       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:01.364843       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:01.364929       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:01,705][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:21:01,706][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:21:01.860830       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (25.101µs)
I0410 09:21:01.860875       1 controller.go:28] processed work item
I0410 09:21:01.860898       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.201µs)
I0410 09:21:01.860912       1 controller.go:28] processed work item
[2018-04-10T09:21:04,707][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:21:05.035005       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:05.035040       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:21:05.976684       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:07.141104       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (43.701µs)
I0410 09:21:07.141165       1 controller.go:28] processed work item
I0410 09:21:07.736160       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (29.701µs)
I0410 09:21:07.736304       1 controller.go:28] processed work item
I0410 09:21:08.040874       1 logs.go:41] http: panic serving 10.0.16.9:44694: runtime error: invalid memory address or nil pointer dereference
goroutine 710 [running]:
net/http.(*conn).serve.func1(0xc4204365a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420427680, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021e2d0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420555b40, 0x1b57140, 0xc420427680, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dde80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc4205ae000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc4205ae000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc4205ae000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204365a0, 0x1b570c0, 0xc420625580)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:21:08.141032       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27µs)
I0410 09:21:08.141085       1 controller.go:28] processed work item
I0410 09:21:08.735943       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17µs)
I0410 09:21:08.735986       1 controller.go:28] processed work item
I0410 09:21:08.740939       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (14.4µs)
I0410 09:21:08.740979       1 controller.go:28] processed work item
I0410 09:21:08.745450       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:08.745478       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:08.941018       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.4µs)
I0410 09:21:08.941068       1 controller.go:28] processed work item
I0410 09:21:09.336027       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.401µs)
I0410 09:21:09.336074       1 controller.go:28] processed work item
I0410 09:21:09.336109       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.401µs)
I0410 09:21:09.336124       1 controller.go:28] processed work item
[2018-04-10T09:21:09,708][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:21:10.533489       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:21:10.805609       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (27.101µs)
I0410 09:21:10.805653       1 controller.go:28] processed work item
E0410 09:21:10.986344       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:11.866347       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.9µs)
I0410 09:21:11.866390       1 controller.go:28] processed work item
I0410 09:21:11.866413       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (12.801µs)
I0410 09:21:11.866427       1 controller.go:28] processed work item
I0410 09:21:12.103683       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:12.103713       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:12,708][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:21:15.667272       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:15.667317       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:21:15.995179       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:21:17,709][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:21:17.741562       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.7µs)
I0410 09:21:17.741604       1 controller.go:28] processed work item
I0410 09:21:18.040821       1 logs.go:41] http: panic serving 10.0.16.9:44872: runtime error: invalid memory address or nil pointer dereference
goroutine 719 [running]:
net/http.(*conn).serve.func1(0xc420436960)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4203b08a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021f530, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420555b40, 0x1b57140, 0xc4203b08a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42035d980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c2a0, 0xc4205aef00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c2a0, 0xc4205aef00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c2a0, 0xc4205aef00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436960, 0x1b570c0, 0xc42008c400)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:21:18.741283       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.8µs)
I0410 09:21:18.741330       1 controller.go:28] processed work item
I0410 09:21:18.996688       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:18.996722       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:19.335661       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.9µs)
I0410 09:21:19.335710       1 controller.go:28] processed work item
I0410 09:21:19.341381       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.3µs)
I0410 09:21:19.341423       1 controller.go:28] processed work item
I0410 09:21:19.341443       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (9.301µs)
I0410 09:21:19.341467       1 controller.go:28] processed work item
I0410 09:21:19.936114       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.001µs)
I0410 09:21:19.936156       1 controller.go:28] processed work item
I0410 09:21:19.936180       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11.7µs)
I0410 09:21:19.936193       1 controller.go:28] processed work item
[2018-04-10T09:21:20,710][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:21:20.811009       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.601µs)
I0410 09:21:20.811082       1 controller.go:28] processed work item
E0410 09:21:21.003761       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:21.871807       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.801µs)
I0410 09:21:21.871861       1 controller.go:28] processed work item
I0410 09:21:21.871884       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11.8µs)
I0410 09:21:21.871898       1 controller.go:28] processed work item
I0410 09:21:22.815717       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:22.815786       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:24,726][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:21:25.789627       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:25.789672       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:21:26.012510       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:21:27,727][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:21:27,728][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:21:27.746943       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.401µs)
I0410 09:21:27.746985       1 controller.go:28] processed work item
I0410 09:21:28.040938       1 logs.go:41] http: panic serving 10.0.16.9:45030: runtime error: invalid memory address or nil pointer dereference
goroutine 749 [running]:
net/http.(*conn).serve.func1(0xc4206c8fa0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420514120, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420291350, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420788b40, 0x1b57140, 0xc420514120, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8540, 0xc42076d500)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8540, 0xc42076d500)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8540, 0xc42076d500)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c8fa0, 0x1b570c0, 0xc420261240)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:21:28.106987       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:28.107032       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:29.341033       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.9µs)
I0410 09:21:29.341086       1 controller.go:28] processed work item
I0410 09:21:29.935908       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.901µs)
I0410 09:21:29.935960       1 controller.go:28] processed work item
I0410 09:21:29.941497       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (13.401µs)
I0410 09:21:29.941529       1 controller.go:28] processed work item
I0410 09:21:29.941575       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.7µs)
I0410 09:21:29.941586       1 controller.go:28] processed work item
I0410 09:21:30.535779       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (42.301µs)
I0410 09:21:30.535848       1 controller.go:28] processed work item
I0410 09:21:30.535872       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.8µs)
I0410 09:21:30.536005       1 controller.go:28] processed work item
[2018-04-10T09:21:30,729][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:21:30,730][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:21:30.816409       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.301µs)
I0410 09:21:30.816454       1 controller.go:28] processed work item
E0410 09:21:31.020360       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:31.877229       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.301µs)
I0410 09:21:31.877272       1 controller.go:28] processed work item
I0410 09:21:31.877293       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.9µs)
I0410 09:21:31.877306       1 controller.go:28] processed work item
I0410 09:21:32.477707       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:32.477752       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:33,731][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:21:33,732][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:21:36.028720       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:36.633465       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:36.633517       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:36,733][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:21:37.752403       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.2µs)
I0410 09:21:37.752449       1 controller.go:28] processed work item
I0410 09:21:38.043148       1 logs.go:41] http: panic serving 10.0.16.9:45196: runtime error: invalid memory address or nil pointer dereference
goroutine 787 [running]:
net/http.(*conn).serve.func1(0xc4201fcdc0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3c80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b2630, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc4204e3c80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4203e8900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4203460e0, 0xc420147900)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4203460e0, 0xc420147900)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4203460e0, 0xc420147900)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fcdc0, 0x1b570c0, 0xc42060ffc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:21:39.336559       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.901µs)
I0410 09:21:39.336603       1 controller.go:28] processed work item
I0410 09:21:39.410336       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:39.410379       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:39.941360       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (26.601µs)
I0410 09:21:39.941415       1 controller.go:28] processed work item
I0410 09:21:40.533856       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:21:40.535734       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.3µs)
I0410 09:21:40.535774       1 controller.go:28] processed work item
I0410 09:21:40.541176       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.6µs)
I0410 09:21:40.541256       1 controller.go:28] processed work item
I0410 09:21:40.541298       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.401µs)
I0410 09:21:40.541327       1 controller.go:28] processed work item
I0410 09:21:40.821877       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.3µs)
I0410 09:21:40.821927       1 controller.go:28] processed work item
E0410 09:21:41.036586       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:41.135948       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (30.3µs)
I0410 09:21:41.135986       1 controller.go:28] processed work item
I0410 09:21:41.136018       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.4µs)
I0410 09:21:41.136029       1 controller.go:28] processed work item
[2018-04-10T09:21:41,734][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:21:41.882678       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (25.201µs)
I0410 09:21:41.882732       1 controller.go:28] processed work item
I0410 09:21:41.882754       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.3µs)
I0410 09:21:41.882768       1 controller.go:28] processed work item
I0410 09:21:43.145907       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:43.145940       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:44,735][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:21:46.047092       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:46.697945       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:46.697987       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:48.040811       1 logs.go:41] http: panic serving 10.0.16.9:45368: runtime error: invalid memory address or nil pointer dereference
goroutine 781 [running]:
net/http.(*conn).serve.func1(0xc4206c9720)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4206103c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b38f0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420788b40, 0x1b57140, 0xc4206103c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4203462a0, 0xc4206fef00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4203462a0, 0xc4206fef00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4203462a0, 0xc4206fef00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c9720, 0x1b570c0, 0xc420345680)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:21:48.911032       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:48.911079       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:49.341965       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.4µs)
I0410 09:21:49.342005       1 controller.go:28] processed work item
[2018-04-10T09:21:49,735][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:21:50.541208       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (32.501µs)
I0410 09:21:50.541267       1 controller.go:28] processed work item
I0410 09:21:50.827501       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (35.901µs)
I0410 09:21:50.827575       1 controller.go:28] processed work item
E0410 09:21:51.054883       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:21:51.136048       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (29.501µs)
I0410 09:21:51.136110       1 controller.go:28] processed work item
I0410 09:21:51.141394       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.1µs)
I0410 09:21:51.141428       1 controller.go:28] processed work item
I0410 09:21:51.141459       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (9.3µs)
I0410 09:21:51.141471       1 controller.go:28] processed work item
I0410 09:21:51.336058       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.9µs)
I0410 09:21:51.336094       1 controller.go:28] processed work item
I0410 09:21:51.735692       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (30µs)
I0410 09:21:51.735734       1 controller.go:28] processed work item
I0410 09:21:51.888054       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23µs)
I0410 09:21:51.888097       1 controller.go:28] processed work item
I0410 09:21:51.888118       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11µs)
I0410 09:21:51.888131       1 controller.go:28] processed work item
I0410 09:21:51.935497       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.3µs)
I0410 09:21:51.935527       1 controller.go:28] processed work item
I0410 09:21:52.522616       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:52.522643       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:21:52,736][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:21:56.023931       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:56.023964       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:21:56.062036       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:21:56,743][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:21:58.040651       1 logs.go:41] http: panic serving 10.0.16.9:45530: runtime error: invalid memory address or nil pointer dereference
goroutine 822 [running]:
net/http.(*conn).serve.func1(0xc4206c9ae0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420708f00, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c6330, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc420708f00, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5b00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8620, 0xc4206ffb00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8620, 0xc4206ffb00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8620, 0xc4206ffb00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c9ae0, 0x1b570c0, 0xc420345c80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:21:58.916162       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:21:58.916196       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:21:59.347426       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27.101µs)
I0410 09:21:59.347475       1 controller.go:28] processed work item
[2018-04-10T09:21:59,745][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:21:59,746][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:22:00.335719       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.1µs)
I0410 09:22:00.335757       1 controller.go:28] processed work item
I0410 09:22:00.832930       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.9µs)
I0410 09:22:00.832983       1 controller.go:28] processed work item
E0410 09:22:01.071147       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:01.341494       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27.4µs)
I0410 09:22:01.341544       1 controller.go:28] processed work item
I0410 09:22:01.490746       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:01.490777       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:01.741139       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.5µs)
I0410 09:22:01.741195       1 controller.go:28] processed work item
I0410 09:22:01.893462       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (24.4µs)
I0410 09:22:01.893507       1 controller.go:28] processed work item
I0410 09:22:01.893529       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.801µs)
I0410 09:22:01.893543       1 controller.go:28] processed work item
I0410 09:22:01.935672       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.4µs)
I0410 09:22:01.935733       1 controller.go:28] processed work item
I0410 09:22:01.940834       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (13.6µs)
I0410 09:22:01.940864       1 controller.go:28] processed work item
I0410 09:22:02.336008       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.4µs)
I0410 09:22:02.336060       1 controller.go:28] processed work item
I0410 09:22:02.336081       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (9.8µs)
I0410 09:22:02.336093       1 controller.go:28] processed work item
[2018-04-10T09:22:02,747][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:22:02,748][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:22:04.783431       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:04.783469       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:05,749][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:22:05,749][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:22:06.079461       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:07.237875       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:07.237906       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:08.040650       1 logs.go:41] http: panic serving 10.0.16.9:45702: runtime error: invalid memory address or nil pointer dereference
goroutine 829 [running]:
net/http.(*conn).serve.func1(0xc4206c9e00)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4207097a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c6db0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078ab40, 0x1b57140, 0xc4207097a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8700, 0xc4205d0300)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8700, 0xc4205d0300)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8700, 0xc4205d0300)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206c9e00, 0x1b570c0, 0xc4204c61c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:22:08,750][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:22:09.817967       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:09.818013       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:10.341168       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.6µs)
I0410 09:22:10.341236       1 controller.go:28] processed work item
I0410 09:22:10.534135       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:22:10.838353       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (30.701µs)
I0410 09:22:10.838426       1 controller.go:28] processed work item
I0410 09:22:10.935650       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.3µs)
I0410 09:22:10.935693       1 controller.go:28] processed work item
E0410 09:22:11.090130       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:11.898799       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (46.001µs)
I0410 09:22:11.898871       1 controller.go:28] processed work item
I0410 09:22:11.898893       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.7µs)
I0410 09:22:11.898905       1 controller.go:28] processed work item
I0410 09:22:11.941359       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (43.201µs)
I0410 09:22:11.941434       1 controller.go:28] processed work item
I0410 09:22:12.341458       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (26.101µs)
I0410 09:22:12.341496       1 controller.go:28] processed work item
I0410 09:22:12.341517       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (10.3µs)
I0410 09:22:12.341530       1 controller.go:28] processed work item
I0410 09:22:12.535848       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (38.601µs)
I0410 09:22:12.535914       1 controller.go:28] processed work item
I0410 09:22:12.735636       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.901µs)
I0410 09:22:12.735678       1 controller.go:28] processed work item
I0410 09:22:12.935654       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.1µs)
I0410 09:22:12.935694       1 controller.go:28] processed work item
I0410 09:22:13.344119       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:13.344163       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:13,751][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:22:15.653531       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:15.653595       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:22:16.099610       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:22:16,752][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:22:18.040781       1 logs.go:41] http: panic serving 10.0.16.9:45864: runtime error: invalid memory address or nil pointer dereference
goroutine 867 [running]:
net/http.(*conn).serve.func1(0xc4207e2140)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420709e60, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c7bf0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078ab40, 0x1b57140, 0xc420709e60, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2a00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d87e0, 0xc4205d0a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d87e0, 0xc4205d0a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d87e0, 0xc4205d0a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4207e2140, 0x1b570c0, 0xc4204c6880)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:22:18.339363       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:18.339443       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:20.843853       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.601µs)
I0410 09:22:20.843893       1 controller.go:28] processed work item
I0410 09:22:20.941166       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.301µs)
I0410 09:22:20.941204       1 controller.go:28] processed work item
E0410 09:22:21.107325       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:21.334778       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:21.334852       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:21,752][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:22:21.904250       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (55.001µs)
I0410 09:22:21.904298       1 controller.go:28] processed work item
I0410 09:22:21.904320       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.4µs)
I0410 09:22:21.904333       1 controller.go:28] processed work item
I0410 09:22:22.541253       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (45.101µs)
I0410 09:22:22.541298       1 controller.go:28] processed work item
I0410 09:22:22.741154       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (39.101µs)
I0410 09:22:22.741202       1 controller.go:28] processed work item
I0410 09:22:22.941021       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (28.501µs)
I0410 09:22:22.941065       1 controller.go:28] processed work item
I0410 09:22:23.135697       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (20.301µs)
I0410 09:22:23.135740       1 controller.go:28] processed work item
I0410 09:22:23.135770       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.4µs)
I0410 09:22:23.135784       1 controller.go:28] processed work item
I0410 09:22:23.335856       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.3µs)
I0410 09:22:23.335945       1 controller.go:28] processed work item
I0410 09:22:24.383282       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:24.383315       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:24,753][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:22:26.115084       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:27.889114       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:27.889145       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:28.040764       1 logs.go:41] http: panic serving 10.0.16.9:46036: runtime error: invalid memory address or nil pointer dereference
goroutine 862 [running]:
net/http.(*conn).serve.func1(0xc4201fc280)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4203b1020, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fc180, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052bb40, 0x1b57140, 0xc4203b1020, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4203e8e80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8460, 0xc420764600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8460, 0xc420764600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8460, 0xc420764600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc280, 0x1b570c0, 0xc4200530c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:22:28,758][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:22:30.849276       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (74.502µs)
I0410 09:22:30.849326       1 controller.go:28] processed work item
I0410 09:22:30.946492       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.301µs)
I0410 09:22:30.946586       1 controller.go:28] processed work item
E0410 09:22:31.123640       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:31.215170       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:31.215197       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:31,759][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:22:31,761][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:22:31.909654       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.6µs)
I0410 09:22:31.909699       1 controller.go:28] processed work item
I0410 09:22:31.909723       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (12.001µs)
I0410 09:22:31.909748       1 controller.go:28] processed work item
I0410 09:22:33.141073       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.301µs)
I0410 09:22:33.141116       1 controller.go:28] processed work item
I0410 09:22:33.141138       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (10.2µs)
I0410 09:22:33.141152       1 controller.go:28] processed work item
I0410 09:22:33.341244       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (41.701µs)
I0410 09:22:33.341302       1 controller.go:28] processed work item
I0410 09:22:33.736100       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (41.101µs)
I0410 09:22:33.736159       1 controller.go:28] processed work item
I0410 09:22:33.736182       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (11.301µs)
I0410 09:22:33.736196       1 controller.go:28] processed work item
I0410 09:22:33.736294       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (82.602µs)
I0410 09:22:33.736319       1 controller.go:28] processed work item
I0410 09:22:33.935627       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.9µs)
I0410 09:22:33.935678       1 controller.go:28] processed work item
I0410 09:22:34.715657       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:34.715705       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:34,762][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:22:34,762][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:22:36.134232       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:22:37,763][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:22:37,764][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:22:38.040727       1 logs.go:41] http: panic serving 10.0.16.9:46204: runtime error: invalid memory address or nil pointer dereference
goroutine 950 [running]:
net/http.(*conn).serve.func1(0xc420374320)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420057bc0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c64e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc420057bc0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4e00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc4205e7400)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc4205e7400)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc4205e7400)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420374320, 0x1b570c0, 0xc42060e880)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:22:38.470861       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:38.470912       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:40.534513       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
[2018-04-10T09:22:40,765][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:22:40.854787       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.2µs)
I0410 09:22:40.854832       1 controller.go:28] processed work item
I0410 09:22:40.951957       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (37.501µs)
I0410 09:22:40.951990       1 controller.go:28] processed work item
E0410 09:22:41.142828       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:41.915023       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (37.001µs)
I0410 09:22:41.915078       1 controller.go:28] processed work item
I0410 09:22:41.915103       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11.101µs)
I0410 09:22:41.915116       1 controller.go:28] processed work item
I0410 09:22:42.335743       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.8µs)
I0410 09:22:42.335789       1 controller.go:28] processed work item
I0410 09:22:42.468951       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:42.468995       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:43.741501       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.6µs)
I0410 09:22:43.741561       1 controller.go:28] processed work item
I0410 09:22:43.741688       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.901µs)
I0410 09:22:43.741733       1 controller.go:28] processed work item
I0410 09:22:43.941042       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25µs)
I0410 09:22:43.941086       1 controller.go:28] processed work item
I0410 09:22:44.135850       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (27.801µs)
I0410 09:22:44.135892       1 controller.go:28] processed work item
I0410 09:22:44.135925       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.001µs)
I0410 09:22:44.135940       1 controller.go:28] processed work item
I0410 09:22:44.335999       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (21.901µs)
I0410 09:22:44.336056       1 controller.go:28] processed work item
I0410 09:22:44.336079       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11.2µs)
I0410 09:22:44.336093       1 controller.go:28] processed work item
I0410 09:22:44.477659       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:44.477730       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:45,766][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:22:46.150420       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:48.040779       1 logs.go:41] http: panic serving 10.0.16.9:46370: runtime error: invalid memory address or nil pointer dereference
goroutine 941 [running]:
net/http.(*conn).serve.func1(0xc4204366e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3800, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420290c30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078ab40, 0x1b57140, 0xc4204e3800, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420320700, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202e4000, 0xc4205d1700)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202e4000, 0xc4205d1700)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202e4000, 0xc4205d1700)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204366e0, 0x1b570c0, 0xc42008df80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:22:48.249490       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:48.249530       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:22:48,766][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:22:50.860248       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (66.501µs)
I0410 09:22:50.860290       1 controller.go:28] processed work item
E0410 09:22:51.159675       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:22:51.214606       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:51.214649       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:51.920341       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.6µs)
I0410 09:22:51.920394       1 controller.go:28] processed work item
I0410 09:22:51.920414       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.3µs)
I0410 09:22:51.920426       1 controller.go:28] processed work item
I0410 09:22:52.341141       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (33.201µs)
I0410 09:22:52.341205       1 controller.go:28] processed work item
I0410 09:22:52.935689       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (45.101µs)
I0410 09:22:52.935753       1 controller.go:28] processed work item
[2018-04-10T09:22:53,767][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:22:54.141285       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (38.301µs)
I0410 09:22:54.141324       1 controller.go:28] processed work item
I0410 09:22:54.341357       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.301µs)
I0410 09:22:54.341416       1 controller.go:28] processed work item
I0410 09:22:54.341441       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (13.5µs)
I0410 09:22:54.341456       1 controller.go:28] processed work item
I0410 09:22:54.414065       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:54.414093       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:54.535657       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18µs)
I0410 09:22:54.535701       1 controller.go:28] processed work item
I0410 09:22:54.735822       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.3µs)
I0410 09:22:54.735864       1 controller.go:28] processed work item
I0410 09:22:54.735886       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.701µs)
I0410 09:22:54.735898       1 controller.go:28] processed work item
I0410 09:22:54.935639       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.701µs)
I0410 09:22:54.935682       1 controller.go:28] processed work item
E0410 09:22:56.167562       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:22:56,768][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:22:57.869122       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:22:57.869168       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:22:58.040890       1 logs.go:41] http: panic serving 10.0.16.9:46530: runtime error: invalid memory address or nil pointer dereference
goroutine 995 [running]:
net/http.(*conn).serve.func1(0xc4204368c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3ce0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202916e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc4204e3ce0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420320e80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202e40e0, 0xc4205d1d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202e40e0, 0xc4205d1d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202e40e0, 0xc4205d1d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204368c0, 0x1b570c0, 0xc42058c480)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:23:00,786][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:00.856523       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:00.856575       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:00.865884       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.101µs)
I0410 09:23:00.865919       1 controller.go:28] processed work item
E0410 09:23:01.175412       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:01.925766       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (35.101µs)
I0410 09:23:01.925823       1 controller.go:28] processed work item
I0410 09:23:01.925845       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.301µs)
I0410 09:23:01.925859       1 controller.go:28] processed work item
I0410 09:23:02.932256       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:02.932303       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:02.941107       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.201µs)
I0410 09:23:02.941163       1 controller.go:28] processed work item
[2018-04-10T09:23:03,787][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:23:03,787][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:04.540978       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.301µs)
I0410 09:23:04.541053       1 controller.go:28] processed work item
I0410 09:23:04.741288       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.301µs)
I0410 09:23:04.741354       1 controller.go:28] processed work item
I0410 09:23:04.941098       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (21.001µs)
I0410 09:23:04.941128       1 controller.go:28] processed work item
I0410 09:23:04.941246       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:04.941278       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:05.135618       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.8µs)
I0410 09:23:05.135724       1 controller.go:28] processed work item
I0410 09:23:05.336100       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19µs)
I0410 09:23:05.336131       1 controller.go:28] processed work item
I0410 09:23:05.536237       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (20.301µs)
I0410 09:23:05.536272       1 controller.go:28] processed work item
I0410 09:23:05.536296       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12µs)
I0410 09:23:05.536309       1 controller.go:28] processed work item
E0410 09:23:06.183500       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:23:06,789][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:23:06,790][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:06.967081       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:06.967118       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:08.040742       1 logs.go:41] http: panic serving 10.0.16.9:46706: runtime error: invalid memory address or nil pointer dereference
goroutine 976 [running]:
net/http.(*conn).serve.func1(0xc420374f00)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420457a40, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420074780, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078bb40, 0x1b57140, 0xc420457a40, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043b600, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c460, 0xc4201f9d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c460, 0xc4201f9d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c460, 0xc4201f9d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420374f00, 0x1b570c0, 0xc42049c000)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:23:09,791][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:23:09,791][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:10.534814       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:23:10.871256       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (42.901µs)
I0410 09:23:10.871312       1 controller.go:28] processed work item
I0410 09:23:11.170260       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:11.170290       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:23:11.191509       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:11.931141       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.9µs)
I0410 09:23:11.931182       1 controller.go:28] processed work item
I0410 09:23:11.931202       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.4µs)
I0410 09:23:11.931231       1 controller.go:28] processed work item
[2018-04-10T09:23:12,793][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:23:12.946631       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27.8µs)
I0410 09:23:12.946677       1 controller.go:28] processed work item
I0410 09:23:13.935703       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (19µs)
I0410 09:23:13.935745       1 controller.go:28] processed work item
I0410 09:23:14.590417       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:14.590493       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:15.140986       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.001µs)
I0410 09:23:15.141036       1 controller.go:28] processed work item
I0410 09:23:15.536058       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.8µs)
I0410 09:23:15.536110       1 controller.go:28] processed work item
I0410 09:23:15.541519       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.401µs)
I0410 09:23:15.541557       1 controller.go:28] processed work item
I0410 09:23:15.541663       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (94.102µs)
I0410 09:23:15.541689       1 controller.go:28] processed work item
I0410 09:23:16.136007       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.9µs)
I0410 09:23:16.136047       1 controller.go:28] processed work item
I0410 09:23:16.136069       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.701µs)
I0410 09:23:16.136158       1 controller.go:28] processed work item
E0410 09:23:16.199684       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:16.335841       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.001µs)
I0410 09:23:16.335906       1 controller.go:28] processed work item
[2018-04-10T09:23:17,793][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:23:17.937976       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:17.938009       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:18.040678       1 logs.go:41] http: panic serving 10.0.16.9:46872: runtime error: invalid memory address or nil pointer dereference
goroutine 1021 [running]:
net/http.(*conn).serve.func1(0xc4203754a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420738ae0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203baed0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078bb40, 0x1b57140, 0xc420738ae0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420275900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c620, 0xc420242d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c620, 0xc420242d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c620, 0xc420242d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4203754a0, 0x1b570c0, 0xc42049ca40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:23:20,794][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:23:20.876691       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (61.301µs)
I0410 09:23:20.876749       1 controller.go:28] processed work item
E0410 09:23:21.208002       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:21.899096       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:21.899130       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:21.936804       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (20.8µs)
I0410 09:23:21.936852       1 controller.go:28] processed work item
I0410 09:23:21.936873       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.901µs)
I0410 09:23:21.936886       1 controller.go:28] processed work item
I0410 09:23:23.941118       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (52.201µs)
I0410 09:23:23.941172       1 controller.go:28] processed work item
I0410 09:23:25.541478       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.5µs)
I0410 09:23:25.541529       1 controller.go:28] processed work item
[2018-04-10T09:23:25,795][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:23:25.935800       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.5µs)
I0410 09:23:25.935845       1 controller.go:28] processed work item
I0410 09:23:26.010486       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:26.010520       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:26.141556       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (43.101µs)
I0410 09:23:26.141622       1 controller.go:28] processed work item
E0410 09:23:26.215583       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:26.341360       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (28.9µs)
I0410 09:23:26.341408       1 controller.go:28] processed work item
I0410 09:23:26.735987       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (28.8µs)
I0410 09:23:26.736041       1 controller.go:28] processed work item
I0410 09:23:26.736062       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (10µs)
I0410 09:23:26.736074       1 controller.go:28] processed work item
I0410 09:23:27.137884       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.2µs)
I0410 09:23:27.137934       1 controller.go:28] processed work item
I0410 09:23:28.040925       1 logs.go:41] http: panic serving 10.0.16.9:47032: runtime error: invalid memory address or nil pointer dereference
goroutine 1040 [running]:
net/http.(*conn).serve.func1(0xc420437360)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420709da0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4204b9d70, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420788b40, 0x1b57140, 0xc420709da0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420231d80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202e41c0, 0xc420790200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202e41c0, 0xc420790200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202e41c0, 0xc420790200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437360, 0x1b570c0, 0xc42058dcc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:23:28,795][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:23:29.114971       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:29.115004       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:30.882090       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.6µs)
I0410 09:23:30.882134       1 controller.go:28] processed work item
E0410 09:23:31.223901       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:31.942161       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.1µs)
I0410 09:23:31.942206       1 controller.go:28] processed work item
I0410 09:23:31.942240       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.7µs)
I0410 09:23:31.942264       1 controller.go:28] processed work item
I0410 09:23:32.559869       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:32.559900       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:23:32,810][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:33.946567       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.201µs)
I0410 09:23:33.946613       1 controller.go:28] processed work item
I0410 09:23:34.629476       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:34.629533       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:23:35,811][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:23:35,812][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:35.941266       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (52.301µs)
I0410 09:23:35.941325       1 controller.go:28] processed work item
I0410 09:23:36.135869       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19µs)
I0410 09:23:36.135908       1 controller.go:28] processed work item
E0410 09:23:36.231875       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:36.741380       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.7µs)
I0410 09:23:36.741425       1 controller.go:28] processed work item
I0410 09:23:37.143297       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.901µs)
I0410 09:23:37.143350       1 controller.go:28] processed work item
I0410 09:23:37.335713       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.7µs)
I0410 09:23:37.335754       1 controller.go:28] processed work item
I0410 09:23:37.536010       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.4µs)
I0410 09:23:37.536062       1 controller.go:28] processed work item
I0410 09:23:38.040924       1 logs.go:41] http: panic serving 10.0.16.9:47204: runtime error: invalid memory address or nil pointer dereference
goroutine 875 [running]:
net/http.(*conn).serve.func1(0xc42024c000)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420514540, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203384e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420041b40, 0x1b57140, 0xc420514540, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201be000, 0xc42024e000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201be000, 0xc42024e000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201be000, 0xc42024e000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42024c000, 0x1b570c0, 0xc420344c40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:23:38.663973       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:38.664036       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:23:38,813][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:23:38,813][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:40.535116       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:23:40.887603       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (31µs)
I0410 09:23:40.887657       1 controller.go:28] processed work item
E0410 09:23:41.240556       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:41.267677       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:41.267712       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:23:41,815][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:23:41,815][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:23:41.947511       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (24.7µs)
I0410 09:23:41.947554       1 controller.go:28] processed work item
I0410 09:23:41.947575       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.2µs)
I0410 09:23:41.947610       1 controller.go:28] processed work item
I0410 09:23:43.951963       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (35.2µs)
I0410 09:23:43.952039       1 controller.go:28] processed work item
I0410 09:23:44.812629       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:44.812687       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:23:44,816][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:23:44.935811       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.401µs)
I0410 09:23:44.935854       1 controller.go:28] processed work item
I0410 09:23:46.141277       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (64.501µs)
I0410 09:23:46.141332       1 controller.go:28] processed work item
E0410 09:23:46.248726       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:46.736086       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.001µs)
I0410 09:23:46.736138       1 controller.go:28] processed work item
I0410 09:23:47.341083       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.701µs)
I0410 09:23:47.341124       1 controller.go:28] processed work item
I0410 09:23:47.410778       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:47.410813       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:47.541374       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.401µs)
I0410 09:23:47.541439       1 controller.go:28] processed work item
I0410 09:23:47.937551       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.6µs)
I0410 09:23:47.937601       1 controller.go:28] processed work item
I0410 09:23:47.937633       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.4µs)
I0410 09:23:47.937658       1 controller.go:28] processed work item
I0410 09:23:48.041005       1 logs.go:41] http: panic serving 10.0.16.9:47362: runtime error: invalid memory address or nil pointer dereference
goroutine 1122 [running]:
net/http.(*conn).serve.func1(0xc42024c320)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420515260, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203398c0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc420515260, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2d80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201be0e0, 0xc42024ea00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201be0e0, 0xc42024ea00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201be0e0, 0xc42024ea00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42024c320, 0x1b570c0, 0xc420345940)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:23:49,817][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:23:49.832374       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:49.832405       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:50.892918       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.1µs)
I0410 09:23:50.892967       1 controller.go:28] processed work item
E0410 09:23:51.255997       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:51.953174       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (30.3µs)
I0410 09:23:51.953249       1 controller.go:28] processed work item
I0410 09:23:51.953272       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.2µs)
I0410 09:23:51.953285       1 controller.go:28] processed work item
[2018-04-10T09:23:52,818][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:23:53.262436       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:53.262470       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:54.941384       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.301µs)
I0410 09:23:54.941438       1 controller.go:28] processed work item
I0410 09:23:55.336848       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (19.201µs)
I0410 09:23:55.336907       1 controller.go:28] processed work item
E0410 09:23:56.263903       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:23:56.741611       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (33.001µs)
I0410 09:23:56.741661       1 controller.go:28] processed work item
I0410 09:23:57.222991       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:23:57.223021       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:23:57.335884       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.101µs)
I0410 09:23:57.335923       1 controller.go:28] processed work item
[2018-04-10T09:23:57,818][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:23:57.943090       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (36.501µs)
I0410 09:23:57.943149       1 controller.go:28] processed work item
I0410 09:23:57.943170       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10µs)
I0410 09:23:57.943182       1 controller.go:28] processed work item
I0410 09:23:58.040786       1 logs.go:41] http: panic serving 10.0.16.9:47526: runtime error: invalid memory address or nil pointer dereference
goroutine 1116 [running]:
net/http.(*conn).serve.func1(0xc4200a1040)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420708360, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075e420, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420042b40, 0x1b57140, 0xc420708360, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dc280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c000, 0xc42072e000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c000, 0xc42072e000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c000, 0xc42072e000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4200a1040, 0x1b570c0, 0xc420624300)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:23:58.135805       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.6µs)
I0410 09:23:58.135854       1 controller.go:28] processed work item
I0410 09:23:58.335924       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (31.601µs)
I0410 09:23:58.335965       1 controller.go:28] processed work item
I0410 09:23:58.535901       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19µs)
I0410 09:23:58.535942       1 controller.go:28] processed work item
I0410 09:23:58.736626       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.801µs)
I0410 09:23:58.736670       1 controller.go:28] processed work item
[2018-04-10T09:24:00,819][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:24:00.892637       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:00.892671       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:00.898283       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (19.1µs)
I0410 09:24:00.898316       1 controller.go:28] processed work item
E0410 09:24:01.272002       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:01.958570       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (34.3µs)
I0410 09:24:01.958632       1 controller.go:28] processed work item
I0410 09:24:01.958655       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.7µs)
I0410 09:24:01.958669       1 controller.go:28] processed work item
I0410 09:24:02.970409       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:02.970452       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:24:04,829][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:05.342177       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.001µs)
I0410 09:24:05.342250       1 controller.go:28] processed work item
I0410 09:24:05.935722       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.3µs)
I0410 09:24:05.935765       1 controller.go:28] processed work item
I0410 09:24:06.269827       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:06.269864       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:24:06.280579       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:07.341191       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.301µs)
I0410 09:24:07.341293       1 controller.go:28] processed work item
[2018-04-10T09:24:07,831][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:24:07,831][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:07.935929       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.101µs)
I0410 09:24:07.935973       1 controller.go:28] processed work item
I0410 09:24:08.040792       1 logs.go:41] http: panic serving 10.0.16.9:47704: runtime error: invalid memory address or nil pointer dereference
goroutine 908 [running]:
net/http.(*conn).serve.func1(0xc4204363c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420738ae0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bae40, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc420738ae0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420275980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4204ba000, 0xc4205e6d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4204ba000, 0xc4205e6d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4204ba000, 0xc4205e6d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204363c0, 0x1b570c0, 0xc42060ebc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:24:08.341295       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.9µs)
I0410 09:24:08.341350       1 controller.go:28] processed work item
I0410 09:24:08.742089       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (38.101µs)
I0410 09:24:08.742141       1 controller.go:28] processed work item
I0410 09:24:08.935867       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.7µs)
I0410 09:24:08.936032       1 controller.go:28] processed work item
I0410 09:24:09.335956       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.6µs)
I0410 09:24:09.336010       1 controller.go:28] processed work item
I0410 09:24:10.535382       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:24:10.616345       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:10.616380       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:24:10,833][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:24:10,833][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:10.903639       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.4µs)
I0410 09:24:10.903708       1 controller.go:28] processed work item
E0410 09:24:11.288083       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:11.963930       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (25.101µs)
I0410 09:24:11.963976       1 controller.go:28] processed work item
I0410 09:24:11.963999       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.701µs)
I0410 09:24:11.964013       1 controller.go:28] processed work item
[2018-04-10T09:24:13,834][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:24:13,835][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:14.423250       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:14.423307       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:15.941112       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (43.701µs)
I0410 09:24:15.941157       1 controller.go:28] processed work item
E0410 09:24:16.297342       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:24:16,836][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:24:17.133227       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:17.133259       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:17.941362       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.5µs)
I0410 09:24:17.941416       1 controller.go:28] processed work item
I0410 09:24:18.042407       1 logs.go:41] http: panic serving 10.0.16.9:47870: runtime error: invalid memory address or nil pointer dereference
goroutine 1168 [running]:
net/http.(*conn).serve.func1(0xc4201fc8c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426780, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4201e2fc0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420426780, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043ab00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc42024e200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc42024e200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc42024e200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc8c0, 0x1b570c0, 0xc420625500)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:24:18.335692       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29µs)
I0410 09:24:18.335763       1 controller.go:28] processed work item
I0410 09:24:18.941335       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (36.901µs)
I0410 09:24:18.941377       1 controller.go:28] processed work item
I0410 09:24:19.341332       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (35.501µs)
I0410 09:24:19.341383       1 controller.go:28] processed work item
I0410 09:24:19.535806       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.2µs)
I0410 09:24:19.535856       1 controller.go:28] processed work item
I0410 09:24:19.935742       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.001µs)
I0410 09:24:19.935795       1 controller.go:28] processed work item
I0410 09:24:20.909052       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.201µs)
I0410 09:24:20.909097       1 controller.go:28] processed work item
I0410 09:24:20.945102       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:20.945133       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:24:21.305447       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:24:21,836][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:24:21.969386       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (89.002µs)
I0410 09:24:21.969481       1 controller.go:28] processed work item
I0410 09:24:21.969569       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (20.1µs)
I0410 09:24:21.969620       1 controller.go:28] processed work item
I0410 09:24:23.312523       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:23.312584       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:24:24,837][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:24:25.946493       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (64.102µs)
I0410 09:24:25.946547       1 controller.go:28] processed work item
I0410 09:24:26.172368       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:26.172402       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:24:26.313062       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:28.040726       1 logs.go:41] http: panic serving 10.0.16.9:48032: runtime error: invalid memory address or nil pointer dereference
goroutine 1192 [running]:
net/http.(*conn).serve.func1(0xc4201fcc80)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420427080, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420258690, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420427080, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043b380, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c2a0, 0xc42024f200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c2a0, 0xc42024f200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c2a0, 0xc42024f200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fcc80, 0x1b570c0, 0xc4204c6100)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:24:28.341119       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.601µs)
I0410 09:24:28.341176       1 controller.go:28] processed work item
I0410 09:24:28.935935       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.1µs)
I0410 09:24:28.935986       1 controller.go:28] processed work item
I0410 09:24:29.541229       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.001µs)
I0410 09:24:29.541286       1 controller.go:28] processed work item
[2018-04-10T09:24:29,838][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:24:29.941166       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.3µs)
I0410 09:24:29.941208       1 controller.go:28] processed work item
I0410 09:24:30.135672       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.101µs)
I0410 09:24:30.135714       1 controller.go:28] processed work item
I0410 09:24:30.186383       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:30.186418       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:30.536328       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.9µs)
I0410 09:24:30.536379       1 controller.go:28] processed work item
I0410 09:24:30.914424       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (50.901µs)
I0410 09:24:30.914475       1 controller.go:28] processed work item
E0410 09:24:31.320807       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:31.974758       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.2µs)
I0410 09:24:31.974810       1 controller.go:28] processed work item
I0410 09:24:31.974830       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (8.7µs)
I0410 09:24:31.974854       1 controller.go:28] processed work item
I0410 09:24:32.746829       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:32.746864       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:24:32,838][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:24:35.951938       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (35.001µs)
I0410 09:24:35.951989       1 controller.go:28] processed work item
I0410 09:24:36.258532       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:36.258592       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:24:36.329095       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:24:36,852][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:37.335854       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.5µs)
I0410 09:24:37.335905       1 controller.go:28] processed work item
I0410 09:24:38.040863       1 logs.go:41] http: panic serving 10.0.16.9:48208: runtime error: invalid memory address or nil pointer dereference
goroutine 1215 [running]:
net/http.(*conn).serve.func1(0xc4202c45a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e29c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420290240, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc4204e29c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420320d00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8540, 0xc4202f6200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8540, 0xc4202f6200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8540, 0xc4202f6200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4202c45a0, 0x1b570c0, 0xc420052a00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:24:38.941288       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.5µs)
I0410 09:24:38.941341       1 controller.go:28] processed work item
I0410 09:24:39.459273       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:39.459307       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:39.535935       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.7µs)
I0410 09:24:39.535978       1 controller.go:28] processed work item
[2018-04-10T09:24:39,853][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:24:39,854][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:40.141056       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (28.101µs)
I0410 09:24:40.141100       1 controller.go:28] processed work item
I0410 09:24:40.535765       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:24:40.541919       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.7µs)
I0410 09:24:40.541968       1 controller.go:28] processed work item
I0410 09:24:40.735893       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.6µs)
I0410 09:24:40.735933       1 controller.go:28] processed work item
I0410 09:24:40.919773       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.001µs)
I0410 09:24:40.919819       1 controller.go:28] processed work item
I0410 09:24:40.935729       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.6µs)
I0410 09:24:40.935764       1 controller.go:28] processed work item
I0410 09:24:41.135959       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.1µs)
I0410 09:24:41.136001       1 controller.go:28] processed work item
I0410 09:24:41.335752       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.1µs)
I0410 09:24:41.335794       1 controller.go:28] processed work item
E0410 09:24:41.336581       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:41.679353       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:41.679408       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:41.980179       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (35.2µs)
I0410 09:24:41.980252       1 controller.go:28] processed work item
I0410 09:24:41.980274       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.9µs)
I0410 09:24:41.980296       1 controller.go:28] processed work item
[2018-04-10T09:24:42,855][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:24:42,856][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:24:43.750794       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:43.750824       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:24:45,857][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:24:45,857][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:24:46.343872       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:46.696541       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:46.696576       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:47.341270       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.8µs)
I0410 09:24:47.341315       1 controller.go:28] processed work item
I0410 09:24:48.040792       1 logs.go:41] http: panic serving 10.0.16.9:48368: runtime error: invalid memory address or nil pointer dereference
goroutine 1252 [running]:
net/http.(*conn).serve.func1(0xc4202c4780)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3260, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420291170, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc4204e3260, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420321a80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8620, 0xc4202f6c00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8620, 0xc4202f6c00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8620, 0xc4202f6c00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4202c4780, 0x1b570c0, 0xc420053600)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:24:48,859][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:24:49.541359       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (27.301µs)
I0410 09:24:49.541405       1 controller.go:28] processed work item
I0410 09:24:49.935806       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (35.001µs)
I0410 09:24:49.935845       1 controller.go:28] processed work item
I0410 09:24:50.115939       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:50.115973       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:50.925293       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.301µs)
I0410 09:24:50.925359       1 controller.go:28] processed work item
I0410 09:24:51.141346       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.301µs)
I0410 09:24:51.141405       1 controller.go:28] processed work item
I0410 09:24:51.341188       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (38.001µs)
I0410 09:24:51.341269       1 controller.go:28] processed work item
E0410 09:24:51.352280       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:24:51.735946       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.7µs)
I0410 09:24:51.735989       1 controller.go:28] processed work item
I0410 09:24:51.935957       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (42.9µs)
I0410 09:24:51.936000       1 controller.go:28] processed work item
I0410 09:24:51.985570       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (14.201µs)
I0410 09:24:51.985614       1 controller.go:28] processed work item
I0410 09:24:51.985648       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10µs)
I0410 09:24:51.985660       1 controller.go:28] processed work item
[2018-04-10T09:24:53,859][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:24:54.351318       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:54.351350       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:24:56.359767       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:24:56,860][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:24:57.346623       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.701µs)
I0410 09:24:57.346684       1 controller.go:28] processed work item
I0410 09:24:57.729495       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:24:57.729527       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:24:58.040836       1 logs.go:41] http: panic serving 10.0.16.9:48540: runtime error: invalid memory address or nil pointer dereference
goroutine 1263 [running]:
net/http.(*conn).serve.func1(0xc4202c4c80)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3c80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b2870, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc4204e3c80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d87e0, 0xc4202f7d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d87e0, 0xc4202f7d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d87e0, 0xc4202f7d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4202c4c80, 0x1b570c0, 0xc42058c2c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:24:58.335904       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.7µs)
I0410 09:24:58.335945       1 controller.go:28] processed work item
I0410 09:24:59.941149       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26µs)
I0410 09:24:59.941202       1 controller.go:28] processed work item
I0410 09:25:00.535903       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.801µs)
I0410 09:25:00.535947       1 controller.go:28] processed work item
I0410 09:25:00.930601       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.901µs)
I0410 09:25:00.930645       1 controller.go:28] processed work item
I0410 09:25:01.147065       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:01.147098       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:01.368014       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:01.741325       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.901µs)
I0410 09:25:01.741366       1 controller.go:28] processed work item
[2018-04-10T09:25:01,861][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:25:01.941358       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.1µs)
I0410 09:25:01.941412       1 controller.go:28] processed work item
I0410 09:25:01.990954       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.3µs)
I0410 09:25:01.990997       1 controller.go:28] processed work item
I0410 09:25:01.991019       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.8µs)
I0410 09:25:01.991032       1 controller.go:28] processed work item
I0410 09:25:02.336101       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (20.5µs)
I0410 09:25:02.336145       1 controller.go:28] processed work item
I0410 09:25:02.336167       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.1µs)
I0410 09:25:02.336334       1 controller.go:28] processed work item
I0410 09:25:04.139892       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:04.139942       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:25:04,861][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:25:06.375697       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:07.470607       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:07.470636       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:08.040851       1 logs.go:41] http: panic serving 10.0.16.9:48700: runtime error: invalid memory address or nil pointer dereference
goroutine 1129 [running]:
net/http.(*conn).serve.func1(0xc4205ca0a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4202730e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202685d0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420789b40, 0x1b57140, 0xc4202730e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4206d6000, 0xc420234200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4206d6000, 0xc420234200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4206d6000, 0xc420234200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4205ca0a0, 0x1b570c0, 0xc420608080)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:25:08.341283       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.601µs)
I0410 09:25:08.341321       1 controller.go:28] processed work item
[2018-04-10T09:25:08,871][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:08.935879       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.301µs)
I0410 09:25:08.935918       1 controller.go:28] processed work item
I0410 09:25:10.536034       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:25:10.541451       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.1µs)
I0410 09:25:10.541480       1 controller.go:28] processed work item
I0410 09:25:10.654785       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:10.654817       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:10.935967       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.2µs)
I0410 09:25:10.936023       1 controller.go:28] processed work item
I0410 09:25:11.135695       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.4µs)
I0410 09:25:11.135803       1 controller.go:28] processed work item
E0410 09:25:11.383336       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:11.532859       1 reflector.go:428] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: Watch close - *v1alpha1.ElasticsearchCluster total 0 items received
[2018-04-10T09:25:11,872][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:25:11,873][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:11.996292       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (33.001µs)
I0410 09:25:11.996331       1 controller.go:28] processed work item
I0410 09:25:11.996351       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.5µs)
I0410 09:25:11.996363       1 controller.go:28] processed work item
I0410 09:25:12.341526       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (30.901µs)
I0410 09:25:12.341574       1 controller.go:28] processed work item
I0410 09:25:12.341635       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (47.601µs)
I0410 09:25:12.341653       1 controller.go:28] processed work item
I0410 09:25:12.735990       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.001µs)
I0410 09:25:12.736053       1 controller.go:28] processed work item
I0410 09:25:12.936022       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.3µs)
I0410 09:25:12.936055       1 controller.go:28] processed work item
[2018-04-10T09:25:14,874][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:25:14,874][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:14.959797       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:14.959830       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:16.390790       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:17.233328       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:17.233373       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:25:17,876][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:25:17,876][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:18.043207       1 logs.go:41] http: panic serving 10.0.16.9:48870: runtime error: invalid memory address or nil pointer dereference
goroutine 1316 [running]:
net/http.(*conn).serve.func1(0xc4202c54a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205840c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420339c50, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4205840c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42019a100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d88c0, 0xc420629900)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d88c0, 0xc420629900)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d88c0, 0xc420629900)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4202c54a0, 0x1b570c0, 0xc42058d480)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:25:18.941297       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27.201µs)
I0410 09:25:18.941344       1 controller.go:28] processed work item
[2018-04-10T09:25:20,877][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:25:20.941450       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (38.101µs)
I0410 09:25:20.941483       1 controller.go:28] processed work item
I0410 09:25:21.117850       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:21.117894       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:21.141099       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.8µs)
I0410 09:25:21.141146       1 controller.go:28] processed work item
E0410 09:25:21.398469       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:21.735900       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.8µs)
I0410 09:25:21.735942       1 controller.go:28] processed work item
I0410 09:25:22.001675       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (24.201µs)
I0410 09:25:22.001718       1 controller.go:28] processed work item
I0410 09:25:22.001753       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (12.1µs)
I0410 09:25:22.001852       1 controller.go:28] processed work item
I0410 09:25:22.741768       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (38.001µs)
I0410 09:25:22.741825       1 controller.go:28] processed work item
I0410 09:25:22.941477       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.201µs)
I0410 09:25:22.941521       1 controller.go:28] processed work item
I0410 09:25:23.136077       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.201µs)
I0410 09:25:23.136169       1 controller.go:28] processed work item
I0410 09:25:23.335834       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.801µs)
I0410 09:25:23.335907       1 controller.go:28] processed work item
I0410 09:25:24.066384       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:24.066434       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:25:25,878][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:25:26.384187       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:26.384276       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:26.406085       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:28.040728       1 logs.go:41] http: panic serving 10.0.16.9:49038: runtime error: invalid memory address or nil pointer dereference
goroutine 1322 [running]:
net/http.(*conn).serve.func1(0xc4202c5720)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205845a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075a990, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4205845a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42019a400, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d89a0, 0xc4206e2000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d89a0, 0xc4206e2000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d89a0, 0xc4206e2000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4202c5720, 0x1b570c0, 0xc42058db00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:25:28.844596       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:28.844632       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:25:28,879][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:25:28.946733       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.6µs)
I0410 09:25:28.946816       1 controller.go:28] processed work item
I0410 09:25:30.946890       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.801µs)
I0410 09:25:30.946935       1 controller.go:28] processed work item
E0410 09:25:31.412997       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:31.741307       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.8µs)
I0410 09:25:31.741416       1 controller.go:28] processed work item
I0410 09:25:32.006991       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.8µs)
I0410 09:25:32.007027       1 controller.go:28] processed work item
I0410 09:25:32.007050       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11.601µs)
I0410 09:25:32.007075       1 controller.go:28] processed work item
I0410 09:25:32.335920       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.9µs)
I0410 09:25:32.335962       1 controller.go:28] processed work item
I0410 09:25:32.625439       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:32.625491       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:33.141689       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.601µs)
I0410 09:25:33.141732       1 controller.go:28] processed work item
I0410 09:25:33.341226       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.6µs)
I0410 09:25:33.341281       1 controller.go:28] processed work item
I0410 09:25:33.736064       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.701µs)
I0410 09:25:33.736189       1 controller.go:28] processed work item
[2018-04-10T09:25:33,880][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:25:33.935930       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.6µs)
I0410 09:25:33.936013       1 controller.go:28] processed work item
I0410 09:25:34.136092       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (38.201µs)
I0410 09:25:34.136182       1 controller.go:28] processed work item
I0410 09:25:36.199463       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:36.199496       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:36.420908       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:25:36,880][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:25:38.040783       1 logs.go:41] http: panic serving 10.0.16.9:49202: runtime error: invalid memory address or nil pointer dereference
goroutine 1355 [running]:
net/http.(*conn).serve.func1(0xc4201fc0a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420514120, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420338270, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420042b40, 0x1b57140, 0xc420514120, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420318000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c000, 0xc4202f6200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c000, 0xc4202f6200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c000, 0xc4202f6200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc0a0, 0x1b570c0, 0xc42049c000)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:25:38.439782       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:38.439849       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:38.952114       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.201µs)
I0410 09:25:38.952167       1 controller.go:28] processed work item
I0410 09:25:40.337688       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (19.801µs)
I0410 09:25:40.337730       1 controller.go:28] processed work item
I0410 09:25:40.536405       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
[2018-04-10T09:25:40,895][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:40.952188       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.001µs)
I0410 09:25:40.952253       1 controller.go:28] processed work item
E0410 09:25:41.429373       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:41.693475       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:41.693518       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:42.012377       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.3µs)
I0410 09:25:42.012429       1 controller.go:28] processed work item
I0410 09:25:42.012450       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.5µs)
I0410 09:25:42.012463       1 controller.go:28] processed work item
I0410 09:25:42.341474       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.4µs)
I0410 09:25:42.341527       1 controller.go:28] processed work item
I0410 09:25:42.742507       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.3µs)
I0410 09:25:42.742568       1 controller.go:28] processed work item
[2018-04-10T09:25:43,896][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:25:43,896][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:43.937143       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:43.937195       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:43.941409       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (30.001µs)
I0410 09:25:43.941443       1 controller.go:28] processed work item
I0410 09:25:44.136060       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (31.701µs)
I0410 09:25:44.136113       1 controller.go:28] processed work item
I0410 09:25:44.141342       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.701µs)
I0410 09:25:44.141366       1 controller.go:28] processed work item
I0410 09:25:44.335797       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.901µs)
I0410 09:25:44.335839       1 controller.go:28] processed work item
I0410 09:25:44.536017       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (39.001µs)
I0410 09:25:44.536073       1 controller.go:28] processed work item
I0410 09:25:44.735837       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.8µs)
I0410 09:25:44.735940       1 controller.go:28] processed work item
I0410 09:25:46.307035       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:46.307088       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:46.438475       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:25:46,897][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:25:46,898][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:48.040949       1 logs.go:41] http: panic serving 10.0.16.9:49358: runtime error: invalid memory address or nil pointer dereference
goroutine 1384 [running]:
net/http.(*conn).serve.func1(0xc4206de1e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3500, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42055b710, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206eeb40, 0x1b57140, 0xc4204e3500, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4203e9500, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc420628d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc420628d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc420628d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206de1e0, 0x1b570c0, 0xc420053780)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:25:48.494805       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:48.494843       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:25:49,899][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:25:49,900][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:25:50.343049       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.901µs)
I0410 09:25:50.343096       1 controller.go:28] processed work item
I0410 09:25:50.957636       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.301µs)
I0410 09:25:50.957678       1 controller.go:28] processed work item
I0410 09:25:51.255873       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:51.255910       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:51.445895       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:25:52.017832       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (30.201µs)
I0410 09:25:52.017889       1 controller.go:28] processed work item
I0410 09:25:52.017912       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11.201µs)
I0410 09:25:52.017926       1 controller.go:28] processed work item
I0410 09:25:52.747911       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.4µs)
I0410 09:25:52.747957       1 controller.go:28] processed work item
[2018-04-10T09:25:52,901][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:25:53.135815       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.101µs)
I0410 09:25:53.136014       1 controller.go:28] processed work item
I0410 09:25:53.644517       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:53.644567       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:25:54.341112       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.801µs)
I0410 09:25:54.341160       1 controller.go:28] processed work item
I0410 09:25:54.735969       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (21.601µs)
I0410 09:25:54.736008       1 controller.go:28] processed work item
I0410 09:25:54.741154       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.401µs)
I0410 09:25:54.741204       1 controller.go:28] processed work item
I0410 09:25:55.136091       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.501µs)
I0410 09:25:55.136133       1 controller.go:28] processed work item
I0410 09:25:55.980501       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:55.980743       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:25:56.453143       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:25:57,901][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:25:58.040857       1 logs.go:41] http: panic serving 10.0.16.9:49522: runtime error: invalid memory address or nil pointer dereference
goroutine 1413 [running]:
net/http.(*conn).serve.func1(0xc420436460)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4200578c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b3d40, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206efb40, 0x1b57140, 0xc4200578c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5480, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da000, 0xc4205e7e00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da000, 0xc4205e7e00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da000, 0xc4205e7e00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436460, 0x1b570c0, 0xc42008c980)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:25:58.760498       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:25:58.760566       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:00.348432       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.2µs)
I0410 09:26:00.348473       1 controller.go:28] processed work item
[2018-04-10T09:26:00,902][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:26:00.963052       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (35.701µs)
I0410 09:26:00.963094       1 controller.go:28] processed work item
I0410 09:26:01.336182       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (30.3µs)
I0410 09:26:01.336286       1 controller.go:28] processed work item
E0410 09:26:01.460807       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:02.023400       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (155.703µs)
I0410 09:26:02.023447       1 controller.go:28] processed work item
I0410 09:26:02.023475       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (15.201µs)
I0410 09:26:02.023490       1 controller.go:28] processed work item
I0410 09:26:02.059107       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:02.059137       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:03.141214       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.501µs)
I0410 09:26:03.141269       1 controller.go:28] processed work item
I0410 09:26:03.736271       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (63.401µs)
I0410 09:26:03.736314       1 controller.go:28] processed work item
I0410 09:26:04.741420       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (58.401µs)
I0410 09:26:04.741465       1 controller.go:28] processed work item
I0410 09:26:05.141533       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.6µs)
I0410 09:26:05.141580       1 controller.go:28] processed work item
I0410 09:26:05.336402       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (41.2µs)
I0410 09:26:05.336443       1 controller.go:28] processed work item
I0410 09:26:05.433469       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:05.433520       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:05.535921       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.101µs)
I0410 09:26:05.535956       1 controller.go:28] processed work item
I0410 09:26:05.736112       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.7µs)
I0410 09:26:05.736151       1 controller.go:28] processed work item
I0410 09:26:05.736170       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (9.5µs)
I0410 09:26:05.736182       1 controller.go:28] processed work item
[2018-04-10T09:26:05,903][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:26:06.467948       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:08.040684       1 logs.go:41] http: panic serving 10.0.16.9:49698: runtime error: invalid memory address or nil pointer dereference
goroutine 1401 [running]:
net/http.(*conn).serve.func1(0xc4201fc6e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420515f20, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c6840, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206eeb40, 0x1b57140, 0xc420515f20, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42035cb80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c2a0, 0xc4202f7900)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c2a0, 0xc4202f7900)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c2a0, 0xc4202f7900)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc6e0, 0x1b570c0, 0xc42049cd80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:26:08.670244       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:08.670285       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:26:08,903][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:26:10.536724       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:26:10.968620       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (30.6µs)
I0410 09:26:10.968665       1 controller.go:28] processed work item
I0410 09:26:11.341641       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.101µs)
I0410 09:26:11.341689       1 controller.go:28] processed work item
E0410 09:26:11.475958       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:11.935995       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.201µs)
I0410 09:26:11.936044       1 controller.go:28] processed work item
I0410 09:26:12.028872       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (66.401µs)
I0410 09:26:12.028926       1 controller.go:28] processed work item
I0410 09:26:12.028948       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.4µs)
I0410 09:26:12.028962       1 controller.go:28] processed work item
I0410 09:26:12.316286       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:12.316318       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:26:12,918][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:13.741682       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (41.801µs)
I0410 09:26:13.741724       1 controller.go:28] processed work item
I0410 09:26:14.137078       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.501µs)
I0410 09:26:14.137125       1 controller.go:28] processed work item
I0410 09:26:15.741573       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (72.001µs)
I0410 09:26:15.741619       1 controller.go:28] processed work item
I0410 09:26:15.741643       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12µs)
I0410 09:26:15.741657       1 controller.go:28] processed work item
I0410 09:26:15.887132       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:15.887163       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:26:15,919][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:26:15,919][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:16.135967       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (32.301µs)
I0410 09:26:16.136018       1 controller.go:28] processed work item
I0410 09:26:16.335759       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.401µs)
I0410 09:26:16.335801       1 controller.go:28] processed work item
E0410 09:26:16.482895       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:16.536033       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.4µs)
I0410 09:26:16.536074       1 controller.go:28] processed work item
I0410 09:26:18.040688       1 logs.go:41] http: panic serving 10.0.16.9:49866: runtime error: invalid memory address or nil pointer dereference
goroutine 1465 [running]:
net/http.(*conn).serve.func1(0xc4206ded20)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204279e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202588a0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206efb40, 0x1b57140, 0xc4204279e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043ab00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8620, 0xc4206e3300)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8620, 0xc4206e3300)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8620, 0xc4206e3300)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206ded20, 0x1b570c0, 0xc420624200)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:26:18,921][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:26:18,921][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:19.151282       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:19.151314       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:20.974041       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.201µs)
I0410 09:26:20.974084       1 controller.go:28] processed work item
E0410 09:26:21.490151       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:26:21,922][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:26:21,923][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:21.941552       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (48.201µs)
I0410 09:26:21.941603       1 controller.go:28] processed work item
I0410 09:26:22.034258       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (63.501µs)
I0410 09:26:22.034298       1 controller.go:28] processed work item
I0410 09:26:22.034319       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.2µs)
I0410 09:26:22.034332       1 controller.go:28] processed work item
I0410 09:26:22.725684       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:22.725718       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:24.142455       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.4µs)
I0410 09:26:24.142495       1 controller.go:28] processed work item
I0410 09:26:24.535828       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.201µs)
I0410 09:26:24.535879       1 controller.go:28] processed work item
[2018-04-10T09:26:24,924][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:26:26.141390       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (42.701µs)
I0410 09:26:26.141444       1 controller.go:28] processed work item
I0410 09:26:26.450306       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:26.450338       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:26:26.499324       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:26.541650       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.601µs)
I0410 09:26:26.541698       1 controller.go:28] processed work item
I0410 09:26:26.735812       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.601µs)
I0410 09:26:26.735849       1 controller.go:28] processed work item
I0410 09:26:26.935960       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.2µs)
I0410 09:26:26.936004       1 controller.go:28] processed work item
I0410 09:26:27.136176       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (31.201µs)
I0410 09:26:27.136215       1 controller.go:28] processed work item
I0410 09:26:28.040878       1 logs.go:41] http: panic serving 10.0.16.9:50024: runtime error: invalid memory address or nil pointer dereference
goroutine 1493 [running]:
net/http.(*conn).serve.func1(0xc4206df180)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420611500, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202598c0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206efb40, 0x1b57140, 0xc420611500, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043b280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8700, 0xc420378000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8700, 0xc420378000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8700, 0xc420378000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206df180, 0x1b570c0, 0xc420624a80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:26:29,925][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:26:29.988318       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:29.988352       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:30.979567       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.801µs)
I0410 09:26:30.979609       1 controller.go:28] processed work item
E0410 09:26:31.506625       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:31.947026       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.3µs)
I0410 09:26:31.947075       1 controller.go:28] processed work item
I0410 09:26:32.023545       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:32.023580       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:32.039649       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.801µs)
I0410 09:26:32.039688       1 controller.go:28] processed work item
I0410 09:26:32.039710       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11µs)
I0410 09:26:32.039723       1 controller.go:28] processed work item
[2018-04-10T09:26:32,925][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:26:34.101775       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:34.101808       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:34.541373       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.6µs)
I0410 09:26:34.541423       1 controller.go:28] processed work item
I0410 09:26:35.135956       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.8µs)
I0410 09:26:35.135998       1 controller.go:28] processed work item
I0410 09:26:35.335882       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (30.5µs)
I0410 09:26:35.335919       1 controller.go:28] processed work item
I0410 09:26:36.342262       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:36.342332       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:26:36.513729       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:36.941366       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (64.001µs)
I0410 09:26:36.941408       1 controller.go:28] processed work item
I0410 09:26:37.141540       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.401µs)
I0410 09:26:37.141590       1 controller.go:28] processed work item
I0410 09:26:37.536142       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.6µs)
I0410 09:26:37.536182       1 controller.go:28] processed work item
I0410 09:26:37.736039       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.5µs)
I0410 09:26:37.736085       1 controller.go:28] processed work item
[2018-04-10T09:26:37,926][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:26:38.040857       1 logs.go:41] http: panic serving 10.0.16.9:50186: runtime error: invalid memory address or nil pointer dereference
goroutine 1506 [running]:
net/http.(*conn).serve.func1(0xc4201fcfa0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4207088a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4200757d0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc4207088a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420274980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c460, 0xc420331b00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c460, 0xc420331b00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c460, 0xc420331b00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fcfa0, 0x1b570c0, 0xc42058c400)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:26:39.232691       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:39.232723       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:40.536969       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
[2018-04-10T09:26:40,927][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:26:40.984924       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (35.8µs)
I0410 09:26:40.984968       1 controller.go:28] processed work item
E0410 09:26:41.521372       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:41.952358       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.001µs)
I0410 09:26:41.952402       1 controller.go:28] processed work item
I0410 09:26:42.045131       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.501µs)
I0410 09:26:42.045188       1 controller.go:28] processed work item
I0410 09:26:42.045224       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.401µs)
I0410 09:26:42.045239       1 controller.go:28] processed work item
I0410 09:26:43.220393       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:43.220448       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:43.335828       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17µs)
I0410 09:26:43.335867       1 controller.go:28] processed work item
[2018-04-10T09:26:44,943][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:45.341346       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (44.501µs)
I0410 09:26:45.341389       1 controller.go:28] processed work item
I0410 09:26:45.935969       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (30.701µs)
I0410 09:26:45.936023       1 controller.go:28] processed work item
I0410 09:26:46.062511       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:46.062543       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:26:46.528674       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:47.541499       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (33.901µs)
I0410 09:26:47.541557       1 controller.go:28] processed work item
I0410 09:26:47.741365       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.001µs)
I0410 09:26:47.741410       1 controller.go:28] processed work item
I0410 09:26:47.936038       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.5µs)
I0410 09:26:47.936100       1 controller.go:28] processed work item
[2018-04-10T09:26:47,944][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:26:47,945][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:48.040943       1 logs.go:41] http: panic serving 10.0.16.9:50354: runtime error: invalid memory address or nil pointer dereference
goroutine 1529 [running]:
net/http.(*conn).serve.func1(0xc4204374a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420739380, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4204b9dd0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420739380, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420231900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da380, 0xc420567200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da380, 0xc420567200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da380, 0xc420567200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204374a0, 0x1b570c0, 0xc420260e80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:26:48.136052       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (30.7µs)
I0410 09:26:48.136156       1 controller.go:28] processed work item
I0410 09:26:48.893702       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:48.893733       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:26:50,946][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:26:50,947][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:50.990237       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.2µs)
I0410 09:26:50.990314       1 controller.go:28] processed work item
I0410 09:26:51.506351       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:51.506513       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:26:51.537549       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:26:52.050494       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (26.2µs)
I0410 09:26:52.050539       1 controller.go:28] processed work item
I0410 09:26:52.050564       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11µs)
I0410 09:26:52.050675       1 controller.go:28] processed work item
I0410 09:26:52.535649       1 reflector.go:428] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: Watch close - *v1alpha1.Pilot total 218 items received
I0410 09:26:53.341237       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (39.501µs)
I0410 09:26:53.341282       1 controller.go:28] processed work item
[2018-04-10T09:26:53,948][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:26:53,948][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:26:54.032247       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:54.032279       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:55.941415       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (32.001µs)
I0410 09:26:55.941457       1 controller.go:28] processed work item
I0410 09:26:56.140898       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.901µs)
I0410 09:26:56.140937       1 controller.go:28] processed work item
E0410 09:26:56.544568       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:26:56,950][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:26:57.001594       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:26:57.001626       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:26:57.941466       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (28.501µs)
I0410 09:26:57.941523       1 controller.go:28] processed work item
I0410 09:26:58.040909       1 logs.go:41] http: panic serving 10.0.16.9:50520: runtime error: invalid memory address or nil pointer dereference
goroutine 1557 [running]:
net/http.(*conn).serve.func1(0xc4201fd7c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420273bc0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4207f8a50, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f1b40, 0x1b57140, 0xc420273bc0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420145b80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c540, 0xc4205b5600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c540, 0xc4205b5600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c540, 0xc4205b5600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd7c0, 0x1b570c0, 0xc42058d9c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:26:58.141460       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.301µs)
I0410 09:26:58.141567       1 controller.go:28] processed work item
I0410 09:26:58.341016       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.2µs)
I0410 09:26:58.341045       1 controller.go:28] processed work item
I0410 09:26:58.540779       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.201µs)
I0410 09:26:58.540819       1 controller.go:28] processed work item
I0410 09:26:58.741101       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.601µs)
I0410 09:26:58.741181       1 controller.go:28] processed work item
I0410 09:27:00.221187       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:00.221233       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:00.995539       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (46.201µs)
I0410 09:27:00.995584       1 controller.go:28] processed work item
E0410 09:27:01.551606       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:27:01,950][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:27:02.055879       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.4µs)
I0410 09:27:02.055924       1 controller.go:28] processed work item
I0410 09:27:02.055966       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (17.5µs)
I0410 09:27:02.056002       1 controller.go:28] processed work item
I0410 09:27:02.636851       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:02.636877       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:03.346647       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.9µs)
I0410 09:27:03.346690       1 controller.go:28] processed work item
I0410 09:27:04.140832       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.301µs)
I0410 09:27:04.140873       1 controller.go:28] processed work item
[2018-04-10T09:27:04,951][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:27:05.436040       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:05.436071       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:06.146352       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.901µs)
I0410 09:27:06.146408       1 controller.go:28] processed work item
E0410 09:27:06.558667       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:06.740766       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.4µs)
I0410 09:27:06.740833       1 controller.go:28] processed work item
I0410 09:27:08.040671       1 logs.go:41] http: panic serving 10.0.16.9:50686: runtime error: invalid memory address or nil pointer dereference
goroutine 1564 [running]:
net/http.(*conn).serve.func1(0xc4201fd9a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205ae180, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4207f9470, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206eeb40, 0x1b57140, 0xc4205ae180, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c620, 0xc4205b5b00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c620, 0xc4205b5b00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c620, 0xc4205b5b00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd9a0, 0x1b570c0, 0xc42041a680)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:27:08.546158       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.101µs)
I0410 09:27:08.546200       1 controller.go:28] processed work item
I0410 09:27:08.746514       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.5µs)
I0410 09:27:08.746558       1 controller.go:28] processed work item
I0410 09:27:09.140897       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.3µs)
I0410 09:27:09.140939       1 controller.go:28] processed work item
I0410 09:27:09.343324       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (21.901µs)
I0410 09:27:09.343382       1 controller.go:28] processed work item
I0410 09:27:09.343404       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.601µs)
I0410 09:27:09.343417       1 controller.go:28] processed work item
I0410 09:27:09.427087       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:09.427131       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:09,952][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:27:10.537284       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:27:11.000912       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.601µs)
I0410 09:27:11.000951       1 controller.go:28] processed work item
E0410 09:27:11.566455       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:12.061279       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.4µs)
I0410 09:27:12.061317       1 controller.go:28] processed work item
I0410 09:27:12.061339       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11.401µs)
I0410 09:27:12.061364       1 controller.go:28] processed work item
[2018-04-10T09:27:12,953][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:27:13.112737       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:13.112822       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:14.146158       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.601µs)
I0410 09:27:14.146203       1 controller.go:28] processed work item
I0410 09:27:14.340911       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.801µs)
I0410 09:27:14.340966       1 controller.go:28] processed work item
I0410 09:27:15.256245       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:15.256277       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:27:16.573776       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:16.746147       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.5µs)
I0410 09:27:16.746287       1 controller.go:28] processed work item
[2018-04-10T09:27:16,963][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:17.344638       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.4µs)
I0410 09:27:17.344680       1 controller.go:28] processed work item
I0410 09:27:18.041178       1 logs.go:41] http: panic serving 10.0.16.9:50844: runtime error: invalid memory address or nil pointer dereference
goroutine 1623 [running]:
net/http.(*conn).serve.func1(0xc42074c1e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420585440, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4207b2750, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052bb40, 0x1b57140, 0xc420585440, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202a7280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da460, 0xc420750400)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da460, 0xc420750400)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da460, 0xc420750400)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42074c1e0, 0x1b570c0, 0xc4205b2c00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:27:19.348679       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (26.5µs)
I0410 09:27:19.348729       1 controller.go:28] processed work item
I0410 09:27:19.348752       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11.2µs)
I0410 09:27:19.348766       1 controller.go:28] processed work item
I0410 09:27:19.659127       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:19.659159       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:19.755705       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.5µs)
I0410 09:27:19.755749       1 controller.go:28] processed work item
I0410 09:27:19.940954       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.4µs)
I0410 09:27:19.941021       1 controller.go:28] processed work item
[2018-04-10T09:27:19,965][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:27:19,965][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:21.006353       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.5µs)
I0410 09:27:21.006409       1 controller.go:28] processed work item
E0410 09:27:21.581004       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:22.066730       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (33.101µs)
I0410 09:27:22.066772       1 controller.go:28] processed work item
I0410 09:27:22.066794       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.001µs)
I0410 09:27:22.066807       1 controller.go:28] processed work item
I0410 09:27:22.651947       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:22.651983       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:22,967][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:27:22,967][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:24.346381       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.9µs)
I0410 09:27:24.346422       1 controller.go:28] processed work item
I0410 09:27:24.926984       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:24.927015       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:25,969][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:27:25,969][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:27:26.587852       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:27.350125       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (27.301µs)
I0410 09:27:27.350171       1 controller.go:28] processed work item
I0410 09:27:27.740747       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.401µs)
I0410 09:27:27.740810       1 controller.go:28] processed work item
I0410 09:27:28.040713       1 logs.go:41] http: panic serving 10.0.16.9:51008: runtime error: invalid memory address or nil pointer dereference
goroutine 1649 [running]:
net/http.(*conn).serve.func1(0xc4204361e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420611b00, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420205800, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420040b40, 0x1b57140, 0xc420611b00, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dd500, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da000, 0xc4202f7700)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da000, 0xc4202f7700)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da000, 0xc4202f7700)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204361e0, 0x1b570c0, 0xc42060ea80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:27:28.806861       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:28.806898       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:28,970][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:27:29.761056       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.1µs)
I0410 09:27:29.761103       1 controller.go:28] processed work item
I0410 09:27:29.946367       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.5µs)
I0410 09:27:29.946423       1 controller.go:28] processed work item
I0410 09:27:30.540885       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.3µs)
I0410 09:27:30.540936       1 controller.go:28] processed work item
I0410 09:27:30.540957       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (9.9µs)
I0410 09:27:30.540969       1 controller.go:28] processed work item
I0410 09:27:31.011755       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.3µs)
I0410 09:27:31.011799       1 controller.go:28] processed work item
I0410 09:27:31.033764       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:31.033820       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:27:31.595110       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:32.072098       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.901µs)
I0410 09:27:32.072141       1 controller.go:28] processed work item
I0410 09:27:32.072163       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.4µs)
I0410 09:27:32.072177       1 controller.go:28] processed work item
I0410 09:27:33.166781       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:33.166812       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:33,971][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:27:34.351813       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27.001µs)
I0410 09:27:34.351860       1 controller.go:28] processed work item
E0410 09:27:36.602345       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:36.886283       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:36.886342       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:36,972][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:27:37.746117       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (39.501µs)
I0410 09:27:37.746161       1 controller.go:28] processed work item
I0410 09:27:38.040623       1 logs.go:41] http: panic serving 10.0.16.9:51174: runtime error: invalid memory address or nil pointer dereference
goroutine 1659 [running]:
net/http.(*conn).serve.func1(0xc420436460)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204564e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bac30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42054eb40, 0x1b57140, 0xc4204564e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202e6580, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da0e0, 0xc4202f7f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da0e0, 0xc4202f7f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da0e0, 0xc4202f7f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436460, 0x1b570c0, 0xc42060f680)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:27:38.340748       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.001µs)
I0410 09:27:38.340795       1 controller.go:28] processed work item
I0410 09:27:39.496065       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:39.496171       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:40.537557       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:27:40.546253       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (43.001µs)
I0410 09:27:40.546291       1 controller.go:28] processed work item
I0410 09:27:40.546314       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (11.2µs)
I0410 09:27:40.546327       1 controller.go:28] processed work item
I0410 09:27:41.017149       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26µs)
I0410 09:27:41.017207       1 controller.go:28] processed work item
I0410 09:27:41.140866       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.9µs)
I0410 09:27:41.140907       1 controller.go:28] processed work item
I0410 09:27:41.140941       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.101µs)
I0410 09:27:41.140954       1 controller.go:28] processed work item
E0410 09:27:41.611372       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:27:41,972][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:27:42.077468       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.801µs)
I0410 09:27:42.077543       1 controller.go:28] processed work item
I0410 09:27:42.077564       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.1µs)
I0410 09:27:42.077577       1 controller.go:28] processed work item
I0410 09:27:43.536866       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:43.536900       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:44.357180       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.8µs)
I0410 09:27:44.357269       1 controller.go:28] processed work item
[2018-04-10T09:27:44,973][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:27:45.340764       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (19.6µs)
I0410 09:27:45.340805       1 controller.go:28] processed work item
E0410 09:27:46.618385       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:47.878895       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:47.878939       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:48.040804       1 logs.go:41] http: panic serving 10.0.16.9:51334: runtime error: invalid memory address or nil pointer dereference
goroutine 1685 [running]:
net/http.(*conn).serve.func1(0xc4201fc8c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4201de8a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021e150, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420555b40, 0x1b57140, 0xc4201de8a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420231d80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc420567a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc420567a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc420567a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc8c0, 0x1b570c0, 0xc420624f00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:27:48.346143       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (36.601µs)
I0410 09:27:48.346206       1 controller.go:28] processed work item
I0410 09:27:48.740777       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.5µs)
I0410 09:27:48.740827       1 controller.go:28] processed work item
[2018-04-10T09:27:48,992][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:50.394338       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:50.394410       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:51.022568       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.801µs)
I0410 09:27:51.022615       1 controller.go:28] processed work item
I0410 09:27:51.146293       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.101µs)
I0410 09:27:51.146337       1 controller.go:28] processed work item
I0410 09:27:51.146359       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (10.6µs)
I0410 09:27:51.146373       1 controller.go:28] processed work item
E0410 09:27:51.625878       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:51.741002       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.101µs)
I0410 09:27:51.741055       1 controller.go:28] processed work item
I0410 09:27:51.741076       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.3µs)
I0410 09:27:51.741089       1 controller.go:28] processed work item
[2018-04-10T09:27:51,993][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:27:51,994][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:52.082868       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.901µs)
I0410 09:27:52.082925       1 controller.go:28] processed work item
I0410 09:27:52.082956       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (19.001µs)
I0410 09:27:52.082969       1 controller.go:28] processed work item
I0410 09:27:52.451818       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:52.451851       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:54,995][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:27:54,995][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:55.346189       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.301µs)
I0410 09:27:55.346243       1 controller.go:28] processed work item
I0410 09:27:55.940790       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.9µs)
I0410 09:27:55.940838       1 controller.go:28] processed work item
E0410 09:27:56.632455       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:27:56.726046       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:56.726105       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:27:57,996][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:27:57,997][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:27:58.040700       1 logs.go:41] http: panic serving 10.0.16.9:51510: runtime error: invalid memory address or nil pointer dereference
goroutine 1704 [running]:
net/http.(*conn).serve.func1(0xc42074caa0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420427200, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4201e3ec0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc420427200, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42035d380, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8620, 0xc420750d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8620, 0xc420750d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8620, 0xc420750d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42074caa0, 0x1b570c0, 0xc420261200)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:27:58.746152       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.701µs)
I0410 09:27:58.746281       1 controller.go:28] processed work item
I0410 09:27:58.954090       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:27:58.954132       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:27:59.357721       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.2µs)
I0410 09:27:59.357762       1 controller.go:28] processed work item
[2018-04-10T09:28:00,998][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:28:01.027979       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.6µs)
I0410 09:28:01.028017       1 controller.go:28] processed work item
E0410 09:28:01.639328       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:01.746371       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.201µs)
I0410 09:28:01.746440       1 controller.go:28] processed work item
I0410 09:28:01.746485       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (31.4µs)
I0410 09:28:01.746522       1 controller.go:28] processed work item
I0410 09:28:02.088282       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (25.401µs)
I0410 09:28:02.088329       1 controller.go:28] processed work item
I0410 09:28:02.088352       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11.4µs)
I0410 09:28:02.088366       1 controller.go:28] processed work item
I0410 09:28:02.341048       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.7µs)
I0410 09:28:02.341103       1 controller.go:28] processed work item
I0410 09:28:02.341125       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (10.6µs)
I0410 09:28:02.341138       1 controller.go:28] processed work item
I0410 09:28:02.511949       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:02.511991       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:05.265394       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:05.265428       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:05.946148       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.601µs)
I0410 09:28:05.946205       1 controller.go:28] processed work item
[2018-04-10T09:28:05,998][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:28:06.646398       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:06.740892       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.7µs)
I0410 09:28:06.740935       1 controller.go:28] processed work item
I0410 09:28:08.040928       1 logs.go:41] http: panic serving 10.0.16.9:51664: runtime error: invalid memory address or nil pointer dereference
goroutine 1733 [running]:
net/http.(*conn).serve.func1(0xc42074d040)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420273200, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420259380, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420526b40, 0x1b57140, 0xc420273200, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420399280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d87e0, 0xc420751e00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d87e0, 0xc420751e00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d87e0, 0xc420751e00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42074d040, 0x1b570c0, 0xc420261f80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:28:08.346781       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:08.346813       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:28:08,999][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:28:09.363090       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.4µs)
I0410 09:28:09.363132       1 controller.go:28] processed work item
I0410 09:28:09.940916       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17µs)
I0410 09:28:09.940969       1 controller.go:28] processed work item
I0410 09:28:10.537850       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:28:11.033437       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (57.602µs)
I0410 09:28:11.033480       1 controller.go:28] processed work item
I0410 09:28:11.520496       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:11.520529       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:28:11.653620       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:12.093669       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.401µs)
I0410 09:28:12.093722       1 controller.go:28] processed work item
I0410 09:28:12.093743       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.7µs)
I0410 09:28:12.093756       1 controller.go:28] processed work item
I0410 09:28:12.346530       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.201µs)
I0410 09:28:12.346572       1 controller.go:28] processed work item
I0410 09:28:12.346595       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.6µs)
I0410 09:28:12.346608       1 controller.go:28] processed work item
I0410 09:28:12.741016       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.801µs)
I0410 09:28:12.741057       1 controller.go:28] processed work item
I0410 09:28:12.940657       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.101µs)
I0410 09:28:12.940696       1 controller.go:28] processed work item
I0410 09:28:13.723942       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:13.723975       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:28:13,999][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:28:16.660481       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:16.746202       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.9µs)
I0410 09:28:16.746281       1 controller.go:28] processed work item
[2018-04-10T09:28:17,000][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:28:17.341172       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:17.341240       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:18.040814       1 logs.go:41] http: panic serving 10.0.16.9:51846: runtime error: invalid memory address or nil pointer dereference
goroutine 1750 [running]:
net/http.(*conn).serve.func1(0xc420437040)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4203b1140, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c7650, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4203b1140, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da1c0, 0xc420382800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da1c0, 0xc420382800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da1c0, 0xc420382800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437040, 0x1b570c0, 0xc420345780)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:28:19.946348       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.2µs)
I0410 09:28:19.946413       1 controller.go:28] processed work item
I0410 09:28:20.306828       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:20.306860       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:20.540802       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.201µs)
I0410 09:28:20.540852       1 controller.go:28] processed work item
[2018-04-10T09:28:21,011][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:28:21.038920       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (33.301µs)
I0410 09:28:21.038971       1 controller.go:28] processed work item
E0410 09:28:21.667542       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:22.098969       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (33.3µs)
I0410 09:28:22.099022       1 controller.go:28] processed work item
I0410 09:28:22.099045       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11.2µs)
I0410 09:28:22.099136       1 controller.go:28] processed work item
I0410 09:28:22.746513       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (39.401µs)
I0410 09:28:22.746571       1 controller.go:28] processed work item
I0410 09:28:22.946029       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.6µs)
I0410 09:28:22.946083       1 controller.go:28] processed work item
I0410 09:28:23.140671       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.701µs)
I0410 09:28:23.140708       1 controller.go:28] processed work item
I0410 09:28:23.540649       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.9µs)
I0410 09:28:23.540688       1 controller.go:28] processed work item
[2018-04-10T09:28:24,013][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:28:24,013][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:28:24.471985       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:24.472029       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:28:26.674949       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:26.751660       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21µs)
I0410 09:28:26.751709       1 controller.go:28] processed work item
[2018-04-10T09:28:27,014][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:28:27,015][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:28:28.040640       1 logs.go:41] http: panic serving 10.0.16.9:52006: runtime error: invalid memory address or nil pointer dereference
goroutine 1757 [running]:
net/http.(*conn).serve.func1(0xc4204372c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205842a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b2750, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4205842a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328800, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201da2a0, 0xc420382f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201da2a0, 0xc420382f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201da2a0, 0xc420382f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204372c0, 0x1b570c0, 0xc42049c280)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:28:28.755949       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:28.755995       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:28:30,016][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:28:30,016][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:28:30.546311       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.6µs)
I0410 09:28:30.546363       1 controller.go:28] processed work item
I0410 09:28:31.044405       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.901µs)
I0410 09:28:31.044449       1 controller.go:28] processed work item
I0410 09:28:31.140719       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.8µs)
I0410 09:28:31.140761       1 controller.go:28] processed work item
I0410 09:28:31.531598       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:31.531639       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:28:31.682607       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:32.104452       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.301µs)
I0410 09:28:32.104495       1 controller.go:28] processed work item
I0410 09:28:32.104517       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.701µs)
I0410 09:28:32.104531       1 controller.go:28] processed work item
[2018-04-10T09:28:33,017][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:28:33.146145       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.3µs)
I0410 09:28:33.146203       1 controller.go:28] processed work item
I0410 09:28:33.541020       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.401µs)
I0410 09:28:33.541073       1 controller.go:28] processed work item
I0410 09:28:33.546068       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27.501µs)
I0410 09:28:33.546112       1 controller.go:28] processed work item
I0410 09:28:33.940647       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.701µs)
I0410 09:28:33.940736       1 controller.go:28] processed work item
I0410 09:28:34.141025       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (19.501µs)
I0410 09:28:34.141086       1 controller.go:28] processed work item
I0410 09:28:34.340829       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.801µs)
I0410 09:28:34.340937       1 controller.go:28] processed work item
I0410 09:28:34.734601       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:34.734635       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:28:36.689977       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:36.757463       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (28.3µs)
I0410 09:28:36.757508       1 controller.go:28] processed work item
I0410 09:28:37.709184       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:37.709208       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:37.940740       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.8µs)
I0410 09:28:37.940771       1 controller.go:28] processed work item
[2018-04-10T09:28:38,018][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:28:38.040738       1 logs.go:41] http: panic serving 10.0.16.9:52176: runtime error: invalid memory address or nil pointer dereference
goroutine 1769 [running]:
net/http.(*conn).serve.func1(0xc42074d720)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420515c80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fd6b0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420526b40, 0x1b57140, 0xc420515c80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d88c0, 0xc4207df600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d88c0, 0xc4207df600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d88c0, 0xc4207df600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42074d720, 0x1b570c0, 0xc4205b2180)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:28:39.762054       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:39.762111       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:40.538129       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
[2018-04-10T09:28:41,019][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:28:41.049939       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (27.9µs)
I0410 09:28:41.049982       1 controller.go:28] processed work item
I0410 09:28:41.146143       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (72.301µs)
I0410 09:28:41.146192       1 controller.go:28] processed work item
I0410 09:28:41.540742       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.1µs)
I0410 09:28:41.540782       1 controller.go:28] processed work item
E0410 09:28:41.696566       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:42.109931       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (31.001µs)
I0410 09:28:42.109985       1 controller.go:28] processed work item
I0410 09:28:42.110007       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.4µs)
I0410 09:28:42.110020       1 controller.go:28] processed work item
I0410 09:28:43.314683       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:43.314719       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:43.945960       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.3µs)
I0410 09:28:43.946001       1 controller.go:28] processed work item
I0410 09:28:44.140809       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.5µs)
I0410 09:28:44.140882       1 controller.go:28] processed work item
I0410 09:28:44.346324       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.401µs)
I0410 09:28:44.346375       1 controller.go:28] processed work item
I0410 09:28:44.540889       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (29.4µs)
I0410 09:28:44.540927       1 controller.go:28] processed work item
[2018-04-10T09:28:46,020][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:28:46.348640       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:46.348673       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:28:46.703134       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:47.946164       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.101µs)
I0410 09:28:47.946245       1 controller.go:28] processed work item
I0410 09:28:48.040841       1 logs.go:41] http: panic serving 10.0.16.9:52332: runtime error: invalid memory address or nil pointer dereference
goroutine 1827 [running]:
net/http.(*conn).serve.func1(0xc42074dae0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205aeba0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4207ba9f0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420526b40, 0x1b57140, 0xc4205aeba0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d89a0, 0xc4201c0100)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d89a0, 0xc4201c0100)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d89a0, 0xc4201c0100)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42074dae0, 0x1b570c0, 0xc4205b3000)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:28:49,020][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:28:49.168204       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:49.168334       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:51.055271       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.9µs)
I0410 09:28:51.055313       1 controller.go:28] processed work item
I0410 09:28:51.546156       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.7µs)
I0410 09:28:51.546193       1 controller.go:28] processed work item
E0410 09:28:51.709503       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:52.115339       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (40.401µs)
I0410 09:28:52.115387       1 controller.go:28] processed work item
I0410 09:28:52.115407       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.6µs)
I0410 09:28:52.115418       1 controller.go:28] processed work item
I0410 09:28:52.140769       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (14.7µs)
I0410 09:28:52.140801       1 controller.go:28] processed work item
[2018-04-10T09:28:53,033][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:28:53.302525       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:53.302578       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:28:54.146242       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (45.601µs)
I0410 09:28:54.146288       1 controller.go:28] processed work item
I0410 09:28:54.540801       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.5µs)
I0410 09:28:54.540839       1 controller.go:28] processed work item
I0410 09:28:54.546432       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (11.9µs)
I0410 09:28:54.546462       1 controller.go:28] processed work item
I0410 09:28:54.740797       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.6µs)
I0410 09:28:54.740832       1 controller.go:28] processed work item
I0410 09:28:54.973123       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (30.8µs)
I0410 09:28:54.973197       1 controller.go:28] processed work item
I0410 09:28:54.973235       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.401µs)
I0410 09:28:54.973251       1 controller.go:28] processed work item
I0410 09:28:55.874917       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:55.874949       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:28:56,034][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:28:56,035][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:28:56.715887       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:28:57.951527       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23µs)
I0410 09:28:57.951576       1 controller.go:28] processed work item
I0410 09:28:58.040884       1 logs.go:41] http: panic serving 10.0.16.9:52508: runtime error: invalid memory address or nil pointer dereference
goroutine 1845 [running]:
net/http.(*conn).serve.func1(0xc4201fdf40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205ade60, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075ac30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420555b40, 0x1b57140, 0xc4205ade60, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42019a980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c620, 0xc42055cb00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c620, 0xc42055cb00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c620, 0xc42055cb00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fdf40, 0x1b570c0, 0xc420609ac0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:28:58.940938       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (15.701µs)
I0410 09:28:58.940984       1 controller.go:28] processed work item
[2018-04-10T09:28:59,036][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:28:59,036][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:28:59.715208       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:28:59.715305       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:01.060648       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (56.102µs)
I0410 09:29:01.060689       1 controller.go:28] processed work item
E0410 09:29:01.722859       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:01.805544       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:01.805577       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:29:02,038][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:29:02,038][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:29:02.120719       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (25.501µs)
I0410 09:29:02.120778       1 controller.go:28] processed work item
I0410 09:29:02.120801       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.5µs)
I0410 09:29:02.120815       1 controller.go:28] processed work item
I0410 09:29:02.146206       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.701µs)
I0410 09:29:02.146273       1 controller.go:28] processed work item
I0410 09:29:02.740854       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.801µs)
I0410 09:29:02.740890       1 controller.go:28] processed work item
I0410 09:29:04.978625       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.9µs)
I0410 09:29:04.978670       1 controller.go:28] processed work item
I0410 09:29:04.978695       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (11.6µs)
I0410 09:29:04.978782       1 controller.go:28] processed work item
[2018-04-10T09:29:05,039][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:29:05.540933       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.001µs)
I0410 09:29:05.540976       1 controller.go:28] processed work item
I0410 09:29:05.540997       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (10.1µs)
I0410 09:29:05.541019       1 controller.go:28] processed work item
I0410 09:29:05.556671       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:05.556714       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:29:06.729748       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:08.040985       1 logs.go:41] http: panic serving 10.0.16.9:52668: runtime error: invalid memory address or nil pointer dereference
goroutine 1076 [running]:
net/http.(*conn).serve.func1(0xc420802000)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e6240, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42062e5a0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206f0b40, 0x1b57140, 0xc4205e6240, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42080a000, 0xc420806000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42080a000, 0xc420806000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42080a000, 0xc420806000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420802000, 0x1b570c0, 0xc42041a880)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:29:08.946321       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.201µs)
I0410 09:29:08.946376       1 controller.go:28] processed work item
I0410 09:29:09.062844       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:09.062890       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:09.740825       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.401µs)
I0410 09:29:09.740874       1 controller.go:28] processed work item
[2018-04-10T09:29:10,041][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:29:10.538467       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:29:11.066063       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.8µs)
I0410 09:29:11.066108       1 controller.go:28] processed work item
E0410 09:29:11.736533       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:12.126315       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (38.601µs)
I0410 09:29:12.126353       1 controller.go:28] processed work item
I0410 09:29:12.126373       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (8.801µs)
I0410 09:29:12.126384       1 controller.go:28] processed work item
I0410 09:29:12.299460       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:12.299498       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:12.746250       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.4µs)
I0410 09:29:12.746307       1 controller.go:28] processed work item
[2018-04-10T09:29:13,042][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:29:13.141113       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.201µs)
I0410 09:29:13.141152       1 controller.go:28] processed work item
I0410 09:29:14.477946       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:14.477977       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:15.546391       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.901µs)
I0410 09:29:15.546454       1 controller.go:28] processed work item
I0410 09:29:15.546477       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (11.701µs)
I0410 09:29:15.546490       1 controller.go:28] processed work item
I0410 09:29:15.940845       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.701µs)
I0410 09:29:15.940891       1 controller.go:28] processed work item
I0410 09:29:16.140767       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.3µs)
I0410 09:29:16.140815       1 controller.go:28] processed work item
I0410 09:29:16.340782       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.8µs)
I0410 09:29:16.340818       1 controller.go:28] processed work item
E0410 09:29:16.743705       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:18.040934       1 logs.go:41] http: panic serving 10.0.16.9:52834: runtime error: invalid memory address or nil pointer dereference
goroutine 1897 [running]:
net/http.(*conn).serve.func1(0xc4201fc460)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e2960, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203399e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206eeb40, 0x1b57140, 0xc4204e2960, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420319980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc420566c00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc420566c00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc420566c00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc460, 0x1b570c0, 0xc420608a40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:29:18,042][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:29:18.220636       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:18.220678       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:19.746196       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (38.501µs)
I0410 09:29:19.746248       1 controller.go:28] processed work item
[2018-04-10T09:29:21,043][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:29:21.071474       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.2µs)
I0410 09:29:21.071535       1 controller.go:28] processed work item
E0410 09:29:21.751310       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:22.131736       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.1µs)
I0410 09:29:22.131783       1 controller.go:28] processed work item
I0410 09:29:22.131802       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.401µs)
I0410 09:29:22.131813       1 controller.go:28] processed work item
I0410 09:29:22.336658       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:22.336702       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:23.146548       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (43.801µs)
I0410 09:29:23.146601       1 controller.go:28] processed work item
I0410 09:29:23.740886       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.3µs)
I0410 09:29:23.740925       1 controller.go:28] processed work item
[2018-04-10T09:29:25,059][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:29:26.146253       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (52.501µs)
I0410 09:29:26.146384       1 controller.go:28] processed work item
I0410 09:29:26.346251       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (39.401µs)
I0410 09:29:26.346284       1 controller.go:28] processed work item
I0410 09:29:26.698014       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:26.698042       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:26.740847       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.501µs)
I0410 09:29:26.740880       1 controller.go:28] processed work item
I0410 09:29:26.740901       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.8µs)
I0410 09:29:26.740914       1 controller.go:28] processed work item
E0410 09:29:26.760294       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:27.140871       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.3µs)
I0410 09:29:27.140908       1 controller.go:28] processed work item
I0410 09:29:28.040664       1 logs.go:41] http: panic serving 10.0.16.9:52994: runtime error: invalid memory address or nil pointer dereference
goroutine 1868 [running]:
net/http.(*conn).serve.func1(0xc420436140)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426000, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c6c30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420042b40, 0x1b57140, 0xc420426000, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328c80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c000, 0xc4202f7400)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c000, 0xc4202f7400)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c000, 0xc4202f7400)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436140, 0x1b570c0, 0xc420345d00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:29:28,060][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:29:28,061][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:29:29.751649       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (38.501µs)
I0410 09:29:29.751698       1 controller.go:28] processed work item
I0410 09:29:30.736779       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:30.736811       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:29:31,062][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:29:31,063][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:29:31.076899       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21µs)
I0410 09:29:31.076950       1 controller.go:28] processed work item
E0410 09:29:31.766949       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:32.137178       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (60.601µs)
I0410 09:29:32.137282       1 controller.go:28] processed work item
I0410 09:29:32.137308       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (12.4µs)
I0410 09:29:32.137323       1 controller.go:28] processed work item
I0410 09:29:33.746304       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (96.202µs)
I0410 09:29:33.746356       1 controller.go:28] processed work item
[2018-04-10T09:29:34,064][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:29:34,065][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:29:34.143126       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.4µs)
I0410 09:29:34.143168       1 controller.go:28] processed work item
I0410 09:29:34.743046       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:34.743076       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:36.746342       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (54.001µs)
I0410 09:29:36.746412       1 controller.go:28] processed work item
E0410 09:29:36.773864       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:29:37,065][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:29:37.146312       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.1µs)
I0410 09:29:37.146376       1 controller.go:28] processed work item
I0410 09:29:37.340791       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (14.801µs)
I0410 09:29:37.340828       1 controller.go:28] processed work item
I0410 09:29:37.342036       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:37.342115       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:37.540986       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.5µs)
I0410 09:29:37.541025       1 controller.go:28] processed work item
I0410 09:29:38.040896       1 logs.go:41] http: panic serving 10.0.16.9:53158: runtime error: invalid memory address or nil pointer dereference
goroutine 1917 [running]:
net/http.(*conn).serve.func1(0xc420802640)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420456540, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fd7a0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206eeb40, 0x1b57140, 0xc420456540, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5200, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42080a2a0, 0xc420629d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42080a2a0, 0xc420629d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42080a2a0, 0xc420629d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420802640, 0x1b570c0, 0xc4204c6f40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:29:39.757066       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.601µs)
I0410 09:29:39.757119       1 controller.go:28] processed work item
I0410 09:29:40.538753       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:29:41.082327       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.601µs)
I0410 09:29:41.082453       1 controller.go:28] processed work item
I0410 09:29:41.141048       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (38.101µs)
I0410 09:29:41.141130       1 controller.go:28] processed work item
I0410 09:29:41.538700       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:41.538729       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:29:41.780455       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:29:42,066][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:29:42.142609       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.001µs)
I0410 09:29:42.142711       1 controller.go:28] processed work item
I0410 09:29:42.142748       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.5µs)
I0410 09:29:42.142817       1 controller.go:28] processed work item
I0410 09:29:43.724863       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:43.724974       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:44.148581       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.101µs)
I0410 09:29:44.148633       1 controller.go:28] processed work item
I0410 09:29:44.340751       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.601µs)
I0410 09:29:44.340799       1 controller.go:28] processed work item
[2018-04-10T09:29:45,067][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:29:46.787026       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:47.346443       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (226.405µs)
I0410 09:29:47.346615       1 controller.go:28] processed work item
I0410 09:29:47.546386       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.4µs)
I0410 09:29:47.546428       1 controller.go:28] processed work item
I0410 09:29:47.733492       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:47.733525       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:29:47.741421       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.401µs)
I0410 09:29:47.741467       1 controller.go:28] processed work item
I0410 09:29:47.940835       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.601µs)
I0410 09:29:47.940884       1 controller.go:28] processed work item
I0410 09:29:48.040758       1 logs.go:41] http: panic serving 10.0.16.9:53322: runtime error: invalid memory address or nil pointer dereference
goroutine 1974 [running]:
net/http.(*conn).serve.func1(0xc4201fd040)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420610d80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420291bc0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420042b40, 0x1b57140, 0xc420610d80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420399800, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8620, 0xc420807400)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8620, 0xc420807400)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8620, 0xc420807400)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd040, 0x1b570c0, 0xc42008da40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:29:50,067][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:29:51.087751       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.601µs)
I0410 09:29:51.087793       1 controller.go:28] processed work item
I0410 09:29:51.146588       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (34.301µs)
I0410 09:29:51.146631       1 controller.go:28] processed work item
I0410 09:29:51.248823       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:51.248854       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:29:51.793804       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:29:52.148067       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.6µs)
I0410 09:29:52.148120       1 controller.go:28] processed work item
I0410 09:29:52.148141       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10µs)
I0410 09:29:52.148153       1 controller.go:28] processed work item
[2018-04-10T09:29:53,068][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:29:54.346137       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22µs)
I0410 09:29:54.346174       1 controller.go:28] processed work item
I0410 09:29:54.940738       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (16.7µs)
I0410 09:29:54.940786       1 controller.go:28] processed work item
I0410 09:29:55.077466       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:55.077498       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:29:56.800955       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:29:57,076][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:29:57.746747       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.8µs)
I0410 09:29:57.746784       1 controller.go:28] processed work item
I0410 09:29:57.946181       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.601µs)
I0410 09:29:57.946281       1 controller.go:28] processed work item
I0410 09:29:58.040848       1 logs.go:41] http: panic serving 10.0.16.9:53488: runtime error: invalid memory address or nil pointer dereference
goroutine 1951 [running]:
net/http.(*conn).serve.func1(0xc4204368c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205ac0c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021f140, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206ebb40, 0x1b57140, 0xc4205ac0c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42035c600, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc4201c0f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc4201c0f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc4201c0f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204368c0, 0x1b570c0, 0xc420261b80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:29:58.140923       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.6µs)
I0410 09:29:58.140962       1 controller.go:28] processed work item
I0410 09:29:58.541931       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.401µs)
I0410 09:29:58.542076       1 controller.go:28] processed work item
I0410 09:29:58.599058       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:29:58.599096       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:30:00,077][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:30:00,079][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:00.836133       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:00.836164       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:01.093242       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (79.102µs)
I0410 09:30:01.093285       1 controller.go:28] processed work item
I0410 09:30:01.151944       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.3µs)
I0410 09:30:01.152054       1 controller.go:28] processed work item
E0410 09:30:01.810328       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:01.940960       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.8µs)
I0410 09:30:01.941045       1 controller.go:28] processed work item
I0410 09:30:02.153455       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (46.701µs)
I0410 09:30:02.153497       1 controller.go:28] processed work item
I0410 09:30:02.153550       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.601µs)
I0410 09:30:02.153562       1 controller.go:28] processed work item
I0410 09:30:02.876046       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:02.876088       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:30:03,080][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:30:03,081][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:04.946124       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.101µs)
I0410 09:30:04.946171       1 controller.go:28] processed work item
I0410 09:30:05.342414       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.001µs)
I0410 09:30:05.342455       1 controller.go:28] processed work item
I0410 09:30:05.540801       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.001µs)
I0410 09:30:05.540852       1 controller.go:28] processed work item
[2018-04-10T09:30:06,082][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:30:06,082][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:06.328164       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:06.328265       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:30:06.817110       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:08.040792       1 logs.go:41] http: panic serving 10.0.16.9:53660: runtime error: invalid memory address or nil pointer dereference
goroutine 1991 [running]:
net/http.(*conn).serve.func1(0xc4202565a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e6180, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42016ca80, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206ecb40, 0x1b57140, 0xc4205e6180, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202e6100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202c40e0, 0xc420217600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202c40e0, 0xc420217600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202c40e0, 0xc420217600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4202565a0, 0x1b570c0, 0xc420624fc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:30:08.146415       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (52.301µs)
I0410 09:30:08.146465       1 controller.go:28] processed work item
I0410 09:30:08.498078       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:08.498119       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:08.540913       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.901µs)
I0410 09:30:08.540983       1 controller.go:28] processed work item
I0410 09:30:08.547326       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.2µs)
I0410 09:30:08.547408       1 controller.go:28] processed work item
I0410 09:30:08.940794       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.001µs)
I0410 09:30:08.940842       1 controller.go:28] processed work item
[2018-04-10T09:30:09,083][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:30:10.538990       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:30:11.098675       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.701µs)
I0410 09:30:11.098739       1 controller.go:28] processed work item
E0410 09:30:11.823839       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:11.946497       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.8µs)
I0410 09:30:11.946535       1 controller.go:28] processed work item
I0410 09:30:12.158979       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (31.5µs)
I0410 09:30:12.159016       1 controller.go:28] processed work item
I0410 09:30:12.159035       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (8.7µs)
I0410 09:30:12.159047       1 controller.go:28] processed work item
I0410 09:30:12.542231       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.8µs)
I0410 09:30:12.542281       1 controller.go:28] processed work item
I0410 09:30:12.897992       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:12.898033       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:30:14,084][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:30:15.546286       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (79.502µs)
I0410 09:30:15.546348       1 controller.go:28] processed work item
I0410 09:30:16.140817       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (28.9µs)
I0410 09:30:16.140878       1 controller.go:28] processed work item
I0410 09:30:16.461464       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:16.461497       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:30:16.831748       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:30:17,084][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:30:18.040697       1 logs.go:41] http: panic serving 10.0.16.9:53820: runtime error: invalid memory address or nil pointer dereference
goroutine 2027 [running]:
net/http.(*conn).serve.func1(0xc4208032c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420584900, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203badb0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206ebb40, 0x1b57140, 0xc420584900, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043b900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42080a380, 0xc4205b0800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42080a380, 0xc4205b0800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42080a380, 0xc4205b0800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4208032c0, 0x1b570c0, 0xc42060f3c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:30:18.546304       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (85.302µs)
I0410 09:30:18.546357       1 controller.go:28] processed work item
I0410 09:30:18.940936       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23µs)
I0410 09:30:18.940978       1 controller.go:28] processed work item
I0410 09:30:18.946072       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (13.7µs)
I0410 09:30:18.946111       1 controller.go:28] processed work item
I0410 09:30:19.540955       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.1µs)
I0410 09:30:19.540997       1 controller.go:28] processed work item
I0410 09:30:20.830387       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:20.830415       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:21.104096       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (30.801µs)
I0410 09:30:21.104140       1 controller.go:28] processed work item
E0410 09:30:21.838838       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:30:22,085][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:30:22.164353       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (31.601µs)
I0410 09:30:22.164393       1 controller.go:28] processed work item
I0410 09:30:22.164436       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.501µs)
I0410 09:30:22.164480       1 controller.go:28] processed work item
I0410 09:30:22.547647       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.7µs)
I0410 09:30:22.547689       1 controller.go:28] processed work item
I0410 09:30:24.838900       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:24.838942       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:30:25,086][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:30:26.146297       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.101µs)
I0410 09:30:26.146341       1 controller.go:28] processed work item
I0410 09:30:26.340846       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.4µs)
I0410 09:30:26.340892       1 controller.go:28] processed work item
E0410 09:30:26.848251       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:27.640633       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:27.640656       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:28.040961       1 logs.go:41] http: panic serving 10.0.16.9:53980: runtime error: invalid memory address or nil pointer dereference
goroutine 2039 [running]:
net/http.(*conn).serve.func1(0xc420437220)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205adbc0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42062e0c0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420041b40, 0x1b57140, 0xc4205adbc0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420275e00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c460, 0xc4205b4d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c460, 0xc4205b4d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c460, 0xc4205b4d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437220, 0x1b570c0, 0xc42041aa80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:30:28.946357       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (20.9µs)
I0410 09:30:28.946410       1 controller.go:28] processed work item
[2018-04-10T09:30:29,105][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:29.540891       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (33.7µs)
I0410 09:30:29.540943       1 controller.go:28] processed work item
I0410 09:30:29.546162       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.6µs)
I0410 09:30:29.546194       1 controller.go:28] processed work item
I0410 09:30:30.140737       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.9µs)
I0410 09:30:30.140785       1 controller.go:28] processed work item
I0410 09:30:31.109510       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (36.801µs)
I0410 09:30:31.109568       1 controller.go:28] processed work item
I0410 09:30:31.232655       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:31.232696       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:30:31.855044       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:30:32,106][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:30:32,106][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:32.169821       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.001µs)
I0410 09:30:32.169871       1 controller.go:28] processed work item
I0410 09:30:32.169892       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.8µs)
I0410 09:30:32.169904       1 controller.go:28] processed work item
I0410 09:30:32.553068       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.701µs)
I0410 09:30:32.553110       1 controller.go:28] processed work item
[2018-04-10T09:30:35,107][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:30:35,108][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:35.531403       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:35.531434       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:36.346272       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.401µs)
I0410 09:30:36.346311       1 controller.go:28] processed work item
E0410 09:30:36.863078       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:36.940878       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15µs)
I0410 09:30:36.940925       1 controller.go:28] processed work item
I0410 09:30:38.040900       1 logs.go:41] http: panic serving 10.0.16.9:54158: runtime error: invalid memory address or nil pointer dereference
goroutine 2083 [running]:
net/http.(*conn).serve.func1(0xc420257220)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e7860, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075f890, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206edb40, 0x1b57140, 0xc4205e7860, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dd700, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202c4380, 0xc4205e3c00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202c4380, 0xc4205e3c00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202c4380, 0xc4205e3c00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420257220, 0x1b570c0, 0xc42049cd80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:30:38,109][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:30:38,110][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:30:38.281199       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:38.281241       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:39.546364       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (35.601µs)
I0410 09:30:39.546430       1 controller.go:28] processed work item
I0410 09:30:40.140974       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.401µs)
I0410 09:30:40.141028       1 controller.go:28] processed work item
I0410 09:30:40.146094       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11.7µs)
I0410 09:30:40.146124       1 controller.go:28] processed work item
I0410 09:30:40.539279       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:30:40.729062       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:40.729094       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:30:40.741310       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (34.3µs)
I0410 09:30:40.741358       1 controller.go:28] processed work item
[2018-04-10T09:30:41,110][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:30:41.114833       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.501µs)
I0410 09:30:41.114873       1 controller.go:28] processed work item
E0410 09:30:41.870632       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:42.175157       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.5µs)
I0410 09:30:42.175205       1 controller.go:28] processed work item
I0410 09:30:42.175239       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.6µs)
I0410 09:30:42.175252       1 controller.go:28] processed work item
I0410 09:30:42.558466       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24µs)
I0410 09:30:42.558522       1 controller.go:28] processed work item
I0410 09:30:43.940923       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.6µs)
I0410 09:30:43.940961       1 controller.go:28] processed work item
I0410 09:30:45.056868       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:45.056910       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:30:46,111][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:30:46.877352       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:46.946194       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.001µs)
I0410 09:30:46.946241       1 controller.go:28] processed work item
I0410 09:30:47.541876       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.3µs)
I0410 09:30:47.541913       1 controller.go:28] processed work item
I0410 09:30:48.040760       1 logs.go:41] http: panic serving 10.0.16.9:54316: runtime error: invalid memory address or nil pointer dereference
goroutine 2115 [running]:
net/http.(*conn).serve.func1(0xc420803b80)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4207da2a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420632750, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206ecb40, 0x1b57140, 0xc4207da2a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202f2900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42080a460, 0xc420730a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42080a460, 0xc420730a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42080a460, 0xc420730a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420803b80, 0x1b570c0, 0xc4205b2ec0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:30:49.061843       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:49.061878       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:30:49,111][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:30:50.146346       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (28.9µs)
I0410 09:30:50.146397       1 controller.go:28] processed work item
I0410 09:30:50.740870       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.7µs)
I0410 09:30:50.740934       1 controller.go:28] processed work item
I0410 09:30:50.746626       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.5µs)
I0410 09:30:50.746661       1 controller.go:28] processed work item
I0410 09:30:51.120343       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.101µs)
I0410 09:30:51.120396       1 controller.go:28] processed work item
I0410 09:30:51.345403       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.8µs)
I0410 09:30:51.345455       1 controller.go:28] processed work item
I0410 09:30:51.809157       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:51.809202       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:30:51.886554       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:30:52.180647       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (31.401µs)
I0410 09:30:52.180716       1 controller.go:28] processed work item
I0410 09:30:52.180756       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (17.901µs)
I0410 09:30:52.180781       1 controller.go:28] processed work item
I0410 09:30:53.946329       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (36.901µs)
I0410 09:30:53.946377       1 controller.go:28] processed work item
[2018-04-10T09:30:54,112][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:30:55.748135       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:55.748166       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:30:56.893047       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:30:57,113][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:30:57.547276       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.501µs)
I0410 09:30:57.547319       1 controller.go:28] processed work item
I0410 09:30:58.040878       1 logs.go:41] http: panic serving 10.0.16.9:54480: runtime error: invalid memory address or nil pointer dereference
goroutine 2095 [running]:
net/http.(*conn).serve.func1(0xc420257720)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc42074c360, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4207b3650, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4206efb40, 0x1b57140, 0xc42074c360, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202a7100, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202c4460, 0xc420733000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202c4460, 0xc420733000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202c4460, 0xc420733000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420257720, 0x1b570c0, 0xc42049dc80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:30:58.141103       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17µs)
I0410 09:30:58.141156       1 controller.go:28] processed work item
I0410 09:30:58.755391       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:30:58.755437       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:00.746290       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.701µs)
I0410 09:31:00.746332       1 controller.go:28] processed work item
I0410 09:31:01.125829       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (35.001µs)
I0410 09:31:01.125877       1 controller.go:28] processed work item
[2018-04-10T09:31:01,130][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:01.340971       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27.101µs)
I0410 09:31:01.340997       1 controller.go:28] processed work item
I0410 09:31:01.350711       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (11.2µs)
I0410 09:31:01.350741       1 controller.go:28] processed work item
E0410 09:31:01.900155       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:01.940858       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (14.801µs)
I0410 09:31:01.940912       1 controller.go:28] processed work item
I0410 09:31:02.186163       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (32.501µs)
I0410 09:31:02.186234       1 controller.go:28] processed work item
I0410 09:31:02.186366       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (108.402µs)
I0410 09:31:02.186419       1 controller.go:28] processed work item
I0410 09:31:02.486147       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:02.486176       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:03.951663       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.5µs)
I0410 09:31:03.951700       1 controller.go:28] processed work item
[2018-04-10T09:31:04,132][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:31:04,132][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:04.541091       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.9µs)
I0410 09:31:04.541143       1 controller.go:28] processed work item
I0410 09:31:05.475962       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:05.475994       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:31:06.906789       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:31:07,134][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:31:07,134][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:08.040671       1 logs.go:41] http: panic serving 10.0.16.9:54640: runtime error: invalid memory address or nil pointer dereference
goroutine 2146 [running]:
net/http.(*conn).serve.func1(0xc4201fc140)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420738360, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42062e4e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420040b40, 0x1b57140, 0xc420738360, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420144200, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc4202f6500)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc4202f6500)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc4202f6500)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc140, 0x1b570c0, 0xc42060e100)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:31:08.146525       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.501µs)
I0410 09:31:08.146571       1 controller.go:28] processed work item
I0410 09:31:08.740929       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.4µs)
I0410 09:31:08.740966       1 controller.go:28] processed work item
I0410 09:31:09.630466       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:09.630508       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:31:10,135][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:31:10,136][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:10.539583       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:31:11.131228       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (56.701µs)
I0410 09:31:11.131270       1 controller.go:28] processed work item
I0410 09:31:11.346381       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.601µs)
I0410 09:31:11.346422       1 controller.go:28] processed work item
E0410 09:31:11.913453       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:11.941176       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.7µs)
I0410 09:31:11.941242       1 controller.go:28] processed work item
I0410 09:31:11.946286       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.601µs)
I0410 09:31:11.946330       1 controller.go:28] processed work item
I0410 09:31:12.191568       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.301µs)
I0410 09:31:12.191676       1 controller.go:28] processed work item
I0410 09:31:12.191714       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.8µs)
I0410 09:31:12.191774       1 controller.go:28] processed work item
I0410 09:31:12.540868       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.101µs)
I0410 09:31:12.540925       1 controller.go:28] processed work item
[2018-04-10T09:31:13,137][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:31:13.935335       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:13.935414       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:14.546632       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.101µs)
I0410 09:31:14.546671       1 controller.go:28] processed work item
I0410 09:31:15.140909       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (31.7µs)
I0410 09:31:15.140964       1 controller.go:28] processed work item
I0410 09:31:15.985159       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:15.985192       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:31:16.920231       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:18.040900       1 logs.go:41] http: panic serving 10.0.16.9:54820: runtime error: invalid memory address or nil pointer dereference
goroutine 2154 [running]:
net/http.(*conn).serve.func1(0xc4201fc460)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420739980, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42062f200, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420040b40, 0x1b57140, 0xc420739980, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420145200, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8460, 0xc4202f6e00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8460, 0xc4202f6e00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8460, 0xc4202f6e00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc460, 0x1b570c0, 0xc42060ee40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:31:18,137][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:31:18.746384       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.7µs)
I0410 09:31:18.746430       1 controller.go:28] processed work item
I0410 09:31:19.140783       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.201µs)
I0410 09:31:19.140911       1 controller.go:28] processed work item
I0410 09:31:19.889316       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:19.889460       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:21.136564       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.801µs)
I0410 09:31:21.136669       1 controller.go:28] processed work item
[2018-04-10T09:31:21,138][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:31:21.926883       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:21.946620       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.201µs)
I0410 09:31:21.946661       1 controller.go:28] processed work item
I0410 09:31:22.196937       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.7µs)
I0410 09:31:22.196978       1 controller.go:28] processed work item
I0410 09:31:22.196998       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.9µs)
I0410 09:31:22.197022       1 controller.go:28] processed work item
I0410 09:31:22.540932       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.4µs)
I0410 09:31:22.540982       1 controller.go:28] processed work item
I0410 09:31:22.546295       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.2µs)
I0410 09:31:22.546357       1 controller.go:28] processed work item
I0410 09:31:22.912083       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:22.912117       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:22.940752       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.1µs)
I0410 09:31:22.940795       1 controller.go:28] processed work item
I0410 09:31:24.952449       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:24.952481       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:25.146319       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.401µs)
I0410 09:31:25.146447       1 controller.go:28] processed work item
[2018-04-10T09:31:26,139][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:31:26.933353       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:27.995258       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:27.995291       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:28.041162       1 logs.go:41] http: panic serving 10.0.16.9:54978: runtime error: invalid memory address or nil pointer dereference
goroutine 2141 [running]:
net/http.(*conn).serve.func1(0xc4206263c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420709740, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075f200, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc420709740, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420231180, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42031e0e0, 0xc420566b00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42031e0e0, 0xc420566b00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42031e0e0, 0xc420566b00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206263c0, 0x1b570c0, 0xc420261140)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:31:29,139][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:31:29.146252       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (41.601µs)
I0410 09:31:29.146310       1 controller.go:28] processed work item
I0410 09:31:29.741250       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (48.901µs)
I0410 09:31:29.741315       1 controller.go:28] processed work item
I0410 09:31:31.141998       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26µs)
I0410 09:31:31.142054       1 controller.go:28] processed work item
E0410 09:31:31.940002       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:32.170935       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:32.170982       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:32.202308       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (17.1µs)
I0410 09:31:32.202351       1 controller.go:28] processed work item
I0410 09:31:32.202372       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.201µs)
I0410 09:31:32.202384       1 controller.go:28] processed work item
I0410 09:31:32.546380       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.301µs)
I0410 09:31:32.546459       1 controller.go:28] processed work item
I0410 09:31:32.946203       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.901µs)
I0410 09:31:32.946265       1 controller.go:28] processed work item
I0410 09:31:33.141161       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (80.702µs)
I0410 09:31:33.141244       1 controller.go:28] processed work item
[2018-04-10T09:31:33,158][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:33.340891       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.6µs)
I0410 09:31:33.340929       1 controller.go:28] processed work item
I0410 09:31:35.151832       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.301µs)
I0410 09:31:35.151879       1 controller.go:28] processed work item
[2018-04-10T09:31:36,159][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:31:36,159][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:36.228532       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:36.228601       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:31:36.946825       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:38.040915       1 logs.go:41] http: panic serving 10.0.16.9:55146: runtime error: invalid memory address or nil pointer dereference
goroutine 2229 [running]:
net/http.(*conn).serve.func1(0xc420626780)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4201de8a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075ffb0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc4201de8a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420320000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42031e1c0, 0xc420567700)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42031e1c0, 0xc420567700)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42031e1c0, 0xc420567700)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420626780, 0x1b570c0, 0xc420261a80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:31:38.336030       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:38.336068       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:31:39,161][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:31:39,161][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:39.746629       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.601µs)
I0410 09:31:39.746686       1 controller.go:28] processed work item
I0410 09:31:40.140995       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.5µs)
I0410 09:31:40.141039       1 controller.go:28] processed work item
I0410 09:31:40.539886       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:31:41.147363       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (28.4µs)
I0410 09:31:41.147420       1 controller.go:28] processed work item
I0410 09:31:41.922200       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:41.922290       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:31:41.953865       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:31:42,162][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:31:42,163][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:31:42.207731       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (25.201µs)
I0410 09:31:42.207779       1 controller.go:28] processed work item
I0410 09:31:42.207804       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (13µs)
I0410 09:31:42.207818       1 controller.go:28] processed work item
I0410 09:31:43.146629       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (37.801µs)
I0410 09:31:43.146683       1 controller.go:28] processed work item
I0410 09:31:43.346292       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27µs)
I0410 09:31:43.346346       1 controller.go:28] processed work item
I0410 09:31:43.741073       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (41.2µs)
I0410 09:31:43.741119       1 controller.go:28] processed work item
I0410 09:31:43.940806       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.1µs)
I0410 09:31:43.940844       1 controller.go:28] processed work item
I0410 09:31:44.143690       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.4µs)
I0410 09:31:44.143741       1 controller.go:28] processed work item
I0410 09:31:44.340930       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (31.7µs)
I0410 09:31:44.340998       1 controller.go:28] processed work item
I0410 09:31:44.762110       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:44.762145       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:45.157197       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27.301µs)
I0410 09:31:45.157399       1 controller.go:28] processed work item
[2018-04-10T09:31:45,164][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:31:46.540781       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (20.601µs)
I0410 09:31:46.540845       1 controller.go:28] processed work item
E0410 09:31:46.960669       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:47.975096       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:47.975128       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:48.040837       1 logs.go:41] http: panic serving 10.0.16.9:55314: runtime error: invalid memory address or nil pointer dereference
goroutine 2258 [running]:
net/http.(*conn).serve.func1(0xc420626d20)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e7920, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4201e30e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc4205e7920, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420321e00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201f80e0, 0xc4207d4d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201f80e0, 0xc4207d4d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201f80e0, 0xc4207d4d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420626d20, 0x1b570c0, 0xc42049c000)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:31:50.146423       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.501µs)
I0410 09:31:50.146480       1 controller.go:28] processed work item
[2018-04-10T09:31:50,165][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:31:50.740919       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (27.201µs)
I0410 09:31:50.740989       1 controller.go:28] processed work item
I0410 09:31:51.152750       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.4µs)
I0410 09:31:51.152818       1 controller.go:28] processed work item
E0410 09:31:51.968045       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:51.995552       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:51.995583       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:52.213248       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (37.901µs)
I0410 09:31:52.213295       1 controller.go:28] processed work item
I0410 09:31:52.213319       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (12.001µs)
I0410 09:31:52.213345       1 controller.go:28] processed work item
I0410 09:31:52.688263       1 hook.go:115] Skipping already executed hook for "WriteConfig" in phase "PreStart"
I0410 09:31:52.688337       1 hook.go:115] Skipping already executed hook for "InstallPlugins" in phase "PreStart"
I0410 09:31:52.688354       1 controller.go:22] ElasticsearchController: syncing current pilot "es-logging-mixed-3"
I0410 09:31:52.695294       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-3" (7.063653ms)
I0410 09:31:52.695326       1 controller.go:49] Error syncing Pilot logging/es-logging-mixed-3, requeuing: local elasticsearch client not available
I0410 09:31:52.695363       1 controller.go:28] processed work item
[2018-04-10T09:31:53,165][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:31:53.746478       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.4µs)
I0410 09:31:53.746535       1 controller.go:28] processed work item
I0410 09:31:54.059594       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:54.059627       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:54.141078       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (20.6µs)
I0410 09:31:54.141113       1 controller.go:28] processed work item
I0410 09:31:54.346310       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.2µs)
I0410 09:31:54.346350       1 controller.go:28] processed work item
I0410 09:31:54.540784       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.601µs)
I0410 09:31:54.540822       1 controller.go:28] processed work item
I0410 09:31:56.362364       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:56.362411       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:31:56.546176       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26µs)
I0410 09:31:56.546242       1 controller.go:28] processed work item
E0410 09:31:56.975596       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:31:57.540837       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.801µs)
I0410 09:31:57.540877       1 controller.go:28] processed work item
I0410 09:31:58.040738       1 logs.go:41] http: panic serving 10.0.16.9:55486: runtime error: invalid memory address or nil pointer dereference
goroutine 2266 [running]:
net/http.(*conn).serve.func1(0xc420627180)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426f00, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420259560, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078db40, 0x1b57140, 0xc420426f00, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420399a00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42031e2a0, 0xc4207d5a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42031e2a0, 0xc4207d5a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42031e2a0, 0xc4207d5a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420627180, 0x1b570c0, 0xc42049cc00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:31:58,166][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:31:58.994451       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:31:58.994503       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:00.746351       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (27.301µs)
I0410 09:32:00.746395       1 controller.go:28] processed work item
I0410 09:32:01.158151       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (28.401µs)
I0410 09:32:01.158240       1 controller.go:28] processed work item
[2018-04-10T09:32:01,167][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:32:01.340950       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.2µs)
I0410 09:32:01.341013       1 controller.go:28] processed work item
E0410 09:32:01.982367       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:02.218694       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.701µs)
I0410 09:32:02.218745       1 controller.go:28] processed work item
I0410 09:32:02.218766       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.501µs)
I0410 09:32:02.218805       1 controller.go:28] processed work item
I0410 09:32:03.003006       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:03.003037       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:04.146457       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.901µs)
I0410 09:32:04.146521       1 controller.go:28] processed work item
I0410 09:32:04.340803       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18µs)
I0410 09:32:04.340840       1 controller.go:28] processed work item
I0410 09:32:04.546205       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (33.901µs)
I0410 09:32:04.546289       1 controller.go:28] processed work item
I0410 09:32:04.740818       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16µs)
I0410 09:32:04.740878       1 controller.go:28] processed work item
[2018-04-10T09:32:05,181][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:05.763852       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:05.763887       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:32:06.990005       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:07.546261       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (52.901µs)
I0410 09:32:07.546315       1 controller.go:28] processed work item
I0410 09:32:07.786307       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:07.786346       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:08.040766       1 logs.go:41] http: panic serving 10.0.16.9:55644: runtime error: invalid memory address or nil pointer dereference
goroutine 2287 [running]:
net/http.(*conn).serve.func1(0xc4204370e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3620, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fcb10, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc4204e3620, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4680, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c460, 0xc420775f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c460, 0xc420775f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c460, 0xc420775f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204370e0, 0x1b570c0, 0xc420608ac0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:32:08.140917       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.9µs)
I0410 09:32:08.140962       1 controller.go:28] processed work item
[2018-04-10T09:32:08,182][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:32:08,183][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:10.540153       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:32:11.163543       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.601µs)
I0410 09:32:11.163588       1 controller.go:28] processed work item
[2018-04-10T09:32:11,185][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:32:11,185][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:11.346406       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.6µs)
I0410 09:32:11.346449       1 controller.go:28] processed work item
I0410 09:32:11.741317       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.2µs)
I0410 09:32:11.741359       1 controller.go:28] processed work item
I0410 09:32:11.949953       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:11.949985       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:32:11.996944       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:12.224163       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (45.201µs)
I0410 09:32:12.224228       1 controller.go:28] processed work item
I0410 09:32:12.224251       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.901µs)
I0410 09:32:12.224264       1 controller.go:28] processed work item
[2018-04-10T09:32:14,186][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:32:14,187][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:14.346374       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.201µs)
I0410 09:32:14.346494       1 controller.go:28] processed work item
I0410 09:32:14.741242       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.9µs)
I0410 09:32:14.741281       1 controller.go:28] processed work item
I0410 09:32:14.746236       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.3µs)
I0410 09:32:14.746271       1 controller.go:28] processed work item
I0410 09:32:14.843455       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:14.843555       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:15.340931       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.4µs)
I0410 09:32:15.340964       1 controller.go:28] processed work item
E0410 09:32:17.003709       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:17.088339       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:17.088374       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:32:17,188][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:32:18.040740       1 logs.go:41] http: panic serving 10.0.16.9:55826: runtime error: invalid memory address or nil pointer dereference
goroutine 2323 [running]:
net/http.(*conn).serve.func1(0xc420736b40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420273c80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420291560, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc420273c80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328580, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201f82a0, 0xc420523f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201f82a0, 0xc420523f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201f82a0, 0xc420523f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420736b40, 0x1b570c0, 0xc42041be40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:32:18.146301       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.001µs)
I0410 09:32:18.146347       1 controller.go:28] processed work item
I0410 09:32:20.636395       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:20.636442       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:21.169002       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.8µs)
I0410 09:32:21.169055       1 controller.go:28] processed work item
I0410 09:32:21.746785       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.901µs)
I0410 09:32:21.746854       1 controller.go:28] processed work item
E0410 09:32:22.010289       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:32:22,189][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:32:22.229612       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (24.7µs)
I0410 09:32:22.229655       1 controller.go:28] processed work item
I0410 09:32:22.229678       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.3µs)
I0410 09:32:22.229691       1 controller.go:28] processed work item
I0410 09:32:22.340971       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.801µs)
I0410 09:32:22.341011       1 controller.go:28] processed work item
I0410 09:32:24.488730       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:24.488764       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:24.746692       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.8µs)
I0410 09:32:24.746740       1 controller.go:28] processed work item
[2018-04-10T09:32:25,189][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:32:25.341110       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (33.5µs)
I0410 09:32:25.341152       1 controller.go:28] processed work item
I0410 09:32:25.346276       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.2µs)
I0410 09:32:25.346310       1 controller.go:28] processed work item
I0410 09:32:25.940985       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.201µs)
I0410 09:32:25.941039       1 controller.go:28] processed work item
E0410 09:32:27.016749       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:28.040756       1 logs.go:41] http: panic serving 10.0.16.9:55980: runtime error: invalid memory address or nil pointer dereference
goroutine 2331 [running]:
net/http.(*conn).serve.func1(0xc420736e60)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205aed80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b2d50, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc4205aed80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328c80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c620, 0xc4205a6b00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c620, 0xc4205a6b00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c620, 0xc4205a6b00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420736e60, 0x1b570c0, 0xc4205b2940)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:32:28.151712       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27µs)
I0410 09:32:28.151784       1 controller.go:28] processed work item
I0410 09:32:28.392895       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:28.392967       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:32:30,190][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:32:31.027324       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:31.027355       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:31.174541       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (79.701µs)
I0410 09:32:31.174585       1 controller.go:28] processed work item
E0410 09:32:32.023261       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:32.234976       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.901µs)
I0410 09:32:32.235019       1 controller.go:28] processed work item
I0410 09:32:32.235040       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.601µs)
I0410 09:32:32.235052       1 controller.go:28] processed work item
I0410 09:32:32.346572       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (46.201µs)
I0410 09:32:32.346613       1 controller.go:28] processed work item
I0410 09:32:32.940967       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20µs)
I0410 09:32:32.941034       1 controller.go:28] processed work item
[2018-04-10T09:32:33,191][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:32:33.865838       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:33.865884       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:35.346472       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.001µs)
I0410 09:32:35.346528       1 controller.go:28] processed work item
I0410 09:32:35.941115       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.401µs)
I0410 09:32:35.941167       1 controller.go:28] processed work item
I0410 09:32:35.946322       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.5µs)
I0410 09:32:35.946355       1 controller.go:28] processed work item
I0410 09:32:36.340989       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.201µs)
I0410 09:32:36.341027       1 controller.go:28] processed work item
I0410 09:32:36.384839       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:36.384870       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:32:37.030466       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:32:37,197][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:38.040767       1 logs.go:41] http: panic serving 10.0.16.9:56152: runtime error: invalid memory address or nil pointer dereference
goroutine 2357 [running]:
net/http.(*conn).serve.func1(0xc420437860)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205af260, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203b3980, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078cb40, 0x1b57140, 0xc4205af260, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5900, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c700, 0xc42059f800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c700, 0xc42059f800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c700, 0xc42059f800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437860, 0x1b570c0, 0xc42008c500)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:32:38.157162       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.3µs)
I0410 09:32:38.157207       1 controller.go:28] processed work item
I0410 09:32:39.741250       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (46.001µs)
I0410 09:32:39.741383       1 controller.go:28] processed work item
[2018-04-10T09:32:40,199][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:32:40,199][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:40.362759       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:40.362793       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:40.540523       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:32:41.180013       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.7µs)
I0410 09:32:41.180081       1 controller.go:28] processed work item
E0410 09:32:42.038025       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:42.240415       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.701µs)
I0410 09:32:42.240456       1 controller.go:28] processed work item
I0410 09:32:42.240476       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (9.3µs)
I0410 09:32:42.240488       1 controller.go:28] processed work item
I0410 09:32:42.946408       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.4µs)
I0410 09:32:42.946522       1 controller.go:28] processed work item
[2018-04-10T09:32:43,201][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:32:43,201][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:43.209847       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:43.209890       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:43.540836       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.1µs)
I0410 09:32:43.540878       1 controller.go:28] processed work item
I0410 09:32:45.946576       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27.4µs)
I0410 09:32:45.946620       1 controller.go:28] processed work item
[2018-04-10T09:32:46,202][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:32:46,203][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:32:46.346349       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (42.701µs)
I0410 09:32:46.346390       1 controller.go:28] processed work item
I0410 09:32:46.541054       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.4µs)
I0410 09:32:46.541195       1 controller.go:28] processed work item
I0410 09:32:46.652312       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:46.652341       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:32:47.046711       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:47.141142       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.301µs)
I0410 09:32:47.141326       1 controller.go:28] processed work item
I0410 09:32:48.042536       1 logs.go:41] http: panic serving 10.0.16.9:56308: runtime error: invalid memory address or nil pointer dereference
goroutine 2383 [running]:
net/http.(*conn).serve.func1(0xc420737400)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4207db3e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203396e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42078bb40, 0x1b57140, 0xc4207db3e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420329d00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201f8460, 0xc4205a7c00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201f8460, 0xc4205a7c00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201f8460, 0xc4205a7c00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420737400, 0x1b570c0, 0xc4205b3b40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:32:49,204][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:32:49.746648       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (37.901µs)
I0410 09:32:49.746709       1 controller.go:28] processed work item
I0410 09:32:50.055248       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:50.055280       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:51.185490       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.801µs)
I0410 09:32:51.185548       1 controller.go:28] processed work item
E0410 09:32:52.143130       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:32:52.245890       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (36.501µs)
I0410 09:32:52.245934       1 controller.go:28] processed work item
I0410 09:32:52.245980       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.1µs)
I0410 09:32:52.245994       1 controller.go:28] processed work item
I0410 09:32:53.056344       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:53.056375       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:53.546189       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.501µs)
I0410 09:32:53.546249       1 controller.go:28] processed work item
I0410 09:32:54.141049       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.301µs)
I0410 09:32:54.141092       1 controller.go:28] processed work item
[2018-04-10T09:32:54,204][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:32:55.348797       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:55.348841       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:56.546528       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (27.4µs)
I0410 09:32:56.546607       1 controller.go:28] processed work item
I0410 09:32:57.141483       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.5µs)
I0410 09:32:57.141538       1 controller.go:28] processed work item
I0410 09:32:57.146582       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.8µs)
I0410 09:32:57.146616       1 controller.go:28] processed work item
E0410 09:32:57.151814       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:32:57,205][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:32:57.740847       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.201µs)
I0410 09:32:57.740906       1 controller.go:28] processed work item
I0410 09:32:58.040802       1 logs.go:41] http: panic serving 10.0.16.9:56482: runtime error: invalid memory address or nil pointer dereference
goroutine 2405 [running]:
net/http.(*conn).serve.func1(0xc420171d60)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4203b1500, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420419f20, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4203b1500, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202a6680, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc420566600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc420566600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc420566600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420171d60, 0x1b570c0, 0xc420608340)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:32:59.541300       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:32:59.541350       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:32:59.752056       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.801µs)
I0410 09:32:59.752111       1 controller.go:28] processed work item
I0410 09:33:00.941007       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.901µs)
I0410 09:33:00.941059       1 controller.go:28] processed work item
I0410 09:33:01.190904       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (56.801µs)
I0410 09:33:01.190944       1 controller.go:28] processed work item
I0410 09:33:01.675503       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:01.675565       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:33:02.158859       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:33:02,205][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:33:02.251352       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.6µs)
I0410 09:33:02.251416       1 controller.go:28] processed work item
I0410 09:33:02.251437       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10µs)
I0410 09:33:02.251450       1 controller.go:28] processed work item
I0410 09:33:04.146484       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (28.2µs)
I0410 09:33:04.146539       1 controller.go:28] processed work item
I0410 09:33:04.229391       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:04.229424       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:04.341105       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.501µs)
I0410 09:33:04.341144       1 controller.go:28] processed work item
[2018-04-10T09:33:05,206][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:33:07.012485       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:07.012537       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:07.146874       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (26.2µs)
I0410 09:33:07.146926       1 controller.go:28] processed work item
E0410 09:33:07.166014       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:07.741255       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (43.001µs)
I0410 09:33:07.741297       1 controller.go:28] processed work item
I0410 09:33:07.746135       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (28.4µs)
I0410 09:33:07.746168       1 controller.go:28] processed work item
I0410 09:33:08.040788       1 logs.go:41] http: panic serving 10.0.16.9:56648: runtime error: invalid memory address or nil pointer dereference
goroutine 2444 [running]:
net/http.(*conn).serve.func1(0xc4201fc460)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e2ae0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fd770, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4204e2ae0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4203e9180, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8460, 0xc420629f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8460, 0xc420629f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8460, 0xc420629f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc460, 0x1b570c0, 0xc42049d240)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:33:08.341051       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.4µs)
I0410 09:33:08.341120       1 controller.go:28] processed work item
[2018-04-10T09:33:09,225][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:09.858558       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:09.858598       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:10.540803       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:33:10.946470       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (59.401µs)
I0410 09:33:10.946544       1 controller.go:28] processed work item
I0410 09:33:11.196336       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.601µs)
I0410 09:33:11.196381       1 controller.go:28] processed work item
I0410 09:33:11.540955       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.501µs)
I0410 09:33:11.541004       1 controller.go:28] processed work item
E0410 09:33:12.173072       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:33:12,227][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:33:12,227][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:12.256799       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.1µs)
I0410 09:33:12.256837       1 controller.go:28] processed work item
I0410 09:33:12.256856       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (8.901µs)
I0410 09:33:12.256867       1 controller.go:28] processed work item
I0410 09:33:12.704253       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:12.704287       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:14.346549       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (37.501µs)
I0410 09:33:14.346591       1 controller.go:28] processed work item
I0410 09:33:14.741052       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.7µs)
I0410 09:33:14.741087       1 controller.go:28] processed work item
[2018-04-10T09:33:15,228][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:33:15,229][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:15.438373       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:15.438409       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:33:17.180477       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:17.746647       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (58.601µs)
I0410 09:33:17.746802       1 controller.go:28] processed work item
I0410 09:33:18.040717       1 logs.go:41] http: panic serving 10.0.16.9:56816: runtime error: invalid memory address or nil pointer dereference
goroutine 2469 [running]:
net/http.(*conn).serve.func1(0xc4201fc780)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3320, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420258720, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc4204e3320, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42035c780, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8620, 0xc4206bea00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8620, 0xc4206bea00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8620, 0xc4206bea00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc780, 0x1b570c0, 0xc42049db80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:33:18.141120       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (29.2µs)
I0410 09:33:18.141161       1 controller.go:28] processed work item
[2018-04-10T09:33:18,230][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:33:18,230][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:18.346402       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.3µs)
I0410 09:33:18.346532       1 controller.go:28] processed work item
I0410 09:33:18.941890       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (49.301µs)
I0410 09:33:18.942014       1 controller.go:28] processed work item
I0410 09:33:19.767778       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:19.767838       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:21.201694       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.501µs)
I0410 09:33:21.201742       1 controller.go:28] processed work item
[2018-04-10T09:33:21,232][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:33:21.547346       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (26.101µs)
I0410 09:33:21.547392       1 controller.go:28] processed work item
E0410 09:33:22.187376       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:22.262208       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (24.6µs)
I0410 09:33:22.262273       1 controller.go:28] processed work item
I0410 09:33:22.262298       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (12.3µs)
I0410 09:33:22.262311       1 controller.go:28] processed work item
I0410 09:33:23.384723       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:23.384768       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:24.746513       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (73.002µs)
I0410 09:33:24.746569       1 controller.go:28] processed work item
I0410 09:33:25.341417       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.101µs)
I0410 09:33:25.341482       1 controller.go:28] processed work item
I0410 09:33:25.889184       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:25.889239       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:33:26,232][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
E0410 09:33:27.194196       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:28.040849       1 logs.go:41] http: panic serving 10.0.16.9:56982: runtime error: invalid memory address or nil pointer dereference
goroutine 2479 [running]:
net/http.(*conn).serve.func1(0xc4201fcc80)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420057e00, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420291b30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052ab40, 0x1b57140, 0xc420057e00, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42035d980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc4206bfa00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc4206bfa00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc4206bfa00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fcc80, 0x1b570c0, 0xc420344d00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:33:28.146606       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (60.601µs)
I0410 09:33:28.146657       1 controller.go:28] processed work item
I0410 09:33:28.741166       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.4µs)
I0410 09:33:28.741262       1 controller.go:28] processed work item
I0410 09:33:28.947346       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.401µs)
I0410 09:33:28.947391       1 controller.go:28] processed work item
[2018-04-10T09:33:29,233][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:33:29.541683       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.6µs)
I0410 09:33:29.541722       1 controller.go:28] processed work item
I0410 09:33:30.205917       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:30.205947       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:31.207205       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (29.7µs)
I0410 09:33:31.207289       1 controller.go:28] processed work item
I0410 09:33:31.552677       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (45.901µs)
I0410 09:33:31.552721       1 controller.go:28] processed work item
E0410 09:33:32.200917       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:32.267866       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (26.9µs)
I0410 09:33:32.267908       1 controller.go:28] processed work item
I0410 09:33:32.267930       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.7µs)
I0410 09:33:32.267955       1 controller.go:28] processed work item
I0410 09:33:32.935994       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:32.936051       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:33:34,233][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:33:35.347533       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.3µs)
I0410 09:33:35.347577       1 controller.go:28] processed work item
I0410 09:33:35.741085       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.1µs)
I0410 09:33:35.741149       1 controller.go:28] processed work item
I0410 09:33:36.879554       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:36.879587       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:33:37.207974       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:33:37,234][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:33:38.040780       1 logs.go:41] http: panic serving 10.0.16.9:57138: runtime error: invalid memory address or nil pointer dereference
goroutine 2513 [running]:
net/http.(*conn).serve.func1(0xc420736b40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426a20, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021fb00, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052bb40, 0x1b57140, 0xc420426a20, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420320d00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201f80e0, 0xc4205a6f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201f80e0, 0xc4205a6f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201f80e0, 0xc4205a6f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420736b40, 0x1b570c0, 0xc420609b00)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:33:38.746574       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24µs)
I0410 09:33:38.746621       1 controller.go:28] processed work item
I0410 09:33:39.210554       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:39.210588       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:39.340907       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.201µs)
I0410 09:33:39.340964       1 controller.go:28] processed work item
I0410 09:33:39.541093       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.9µs)
I0410 09:33:39.541136       1 controller.go:28] processed work item
I0410 09:33:39.547024       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.401µs)
I0410 09:33:39.547065       1 controller.go:28] processed work item
I0410 09:33:39.942475       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.101µs)
I0410 09:33:39.942525       1 controller.go:28] processed work item
I0410 09:33:40.541050       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:33:41.212663       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.7µs)
I0410 09:33:41.212708       1 controller.go:28] processed work item
[2018-04-10T09:33:41,248][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:41.558066       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.6µs)
I0410 09:33:41.558110       1 controller.go:28] processed work item
E0410 09:33:42.214711       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:42.273475       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.001µs)
I0410 09:33:42.273513       1 controller.go:28] processed work item
I0410 09:33:42.273531       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (8.6µs)
I0410 09:33:42.273542       1 controller.go:28] processed work item
I0410 09:33:43.140983       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.6µs)
I0410 09:33:43.141025       1 controller.go:28] processed work item
I0410 09:33:43.489348       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:43.489384       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:33:44,249][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:33:44,250][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:45.746468       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.201µs)
I0410 09:33:45.746506       1 controller.go:28] processed work item
I0410 09:33:46.141097       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.3µs)
I0410 09:33:46.141146       1 controller.go:28] processed work item
I0410 09:33:47.034149       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:47.034174       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:33:47.221873       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:33:47,251][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:33:47,251][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:48.040739       1 logs.go:41] http: panic serving 10.0.16.9:57298: runtime error: invalid memory address or nil pointer dereference
goroutine 2536 [running]:
net/http.(*conn).serve.func1(0xc420437360)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4201def60, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4201e3a10, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc4201def60, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202e7400, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c380, 0xc4202f7d00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c380, 0xc4202f7d00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c380, 0xc4202f7d00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437360, 0x1b570c0, 0xc4204c6740)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:33:49.546560       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (32.001µs)
I0410 09:33:49.546663       1 controller.go:28] processed work item
I0410 09:33:49.741171       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.8µs)
I0410 09:33:49.741306       1 controller.go:28] processed work item
I0410 09:33:49.947856       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.5µs)
I0410 09:33:49.947975       1 controller.go:28] processed work item
[2018-04-10T09:33:50,252][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:33:50,253][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:33:50.541174       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.701µs)
I0410 09:33:50.541236       1 controller.go:28] processed work item
I0410 09:33:51.218117       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (35.801µs)
I0410 09:33:51.218175       1 controller.go:28] processed work item
I0410 09:33:51.327003       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:51.327099       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:33:52.228495       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:52.278822       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (25.501µs)
I0410 09:33:52.278864       1 controller.go:28] processed work item
I0410 09:33:52.278886       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.5µs)
I0410 09:33:52.278900       1 controller.go:28] processed work item
I0410 09:33:53.146404       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.801µs)
I0410 09:33:53.146447       1 controller.go:28] processed work item
[2018-04-10T09:33:53,254][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:33:55.282558       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:55.282591       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:56.146513       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.6µs)
I0410 09:33:56.146554       1 controller.go:28] processed work item
I0410 09:33:56.540985       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.9µs)
I0410 09:33:56.541028       1 controller.go:28] processed work item
E0410 09:33:57.235491       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:33:57.722695       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:33:57.722719       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:33:58.041202       1 logs.go:41] http: panic serving 10.0.16.9:57468: runtime error: invalid memory address or nil pointer dereference
goroutine 2555 [running]:
net/http.(*conn).serve.func1(0xc4201fd680)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4201df4a0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075eb70, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc4201df4a0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dc680, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c540, 0xc420256200)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c540, 0xc420256200)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c540, 0xc420256200)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd680, 0x1b570c0, 0xc420261000)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:33:58,254][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:33:59.746531       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (28.301µs)
I0410 09:33:59.746593       1 controller.go:28] processed work item
I0410 09:34:00.341057       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.401µs)
I0410 09:34:00.341099       1 controller.go:28] processed work item
I0410 09:34:00.546528       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.701µs)
I0410 09:34:00.546600       1 controller.go:28] processed work item
I0410 09:34:01.141469       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18µs)
I0410 09:34:01.141509       1 controller.go:28] processed work item
I0410 09:34:01.223525       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.401µs)
I0410 09:34:01.223598       1 controller.go:28] processed work item
[2018-04-10T09:34:01,255][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:34:01.989690       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:01.989721       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:34:02.243017       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:02.284249       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (32.101µs)
I0410 09:34:02.284322       1 controller.go:28] processed work item
I0410 09:34:02.284347       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11.5µs)
I0410 09:34:02.284362       1 controller.go:28] processed work item
I0410 09:34:03.151755       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.9µs)
I0410 09:34:03.151797       1 controller.go:28] processed work item
I0410 09:34:04.160983       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (34.601µs)
I0410 09:34:04.161029       1 controller.go:28] processed work item
I0410 09:34:05.990260       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:05.990294       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:34:06,255][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:34:06.546418       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (30.401µs)
I0410 09:34:06.546462       1 controller.go:28] processed work item
E0410 09:34:07.250085       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:07.341255       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (44.701µs)
I0410 09:34:07.341300       1 controller.go:28] processed work item
I0410 09:34:08.040868       1 logs.go:41] http: panic serving 10.0.16.9:57644: runtime error: invalid memory address or nil pointer dereference
goroutine 2593 [running]:
net/http.(*conn).serve.func1(0xc420736fa0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4201df9e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075f800, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420552b40, 0x1b57140, 0xc4201df9e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420144700, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c620, 0xc420523100)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c620, 0xc420523100)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c620, 0xc420523100)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420736fa0, 0x1b570c0, 0xc4204c7780)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:34:09.183534       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:09.186435       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:34:09,256][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:34:10.346549       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.901µs)
I0410 09:34:10.346613       1 controller.go:28] processed work item
I0410 09:34:10.541353       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:34:10.628159       1 reflector.go:428] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: Watch close - *v1alpha1.ElasticsearchCluster total 0 items received
I0410 09:34:10.940784       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.2µs)
I0410 09:34:10.940834       1 controller.go:28] processed work item
I0410 09:34:11.146951       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (33.301µs)
I0410 09:34:11.147003       1 controller.go:28] processed work item
I0410 09:34:11.228960       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (18.701µs)
I0410 09:34:11.229007       1 controller.go:28] processed work item
I0410 09:34:11.741189       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (68.101µs)
I0410 09:34:11.741264       1 controller.go:28] processed work item
E0410 09:34:12.257141       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:12.289624       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.6µs)
I0410 09:34:12.289663       1 controller.go:28] processed work item
I0410 09:34:12.289684       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.301µs)
I0410 09:34:12.289696       1 controller.go:28] processed work item
I0410 09:34:13.244144       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:13.244186       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:34:13,267][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:14.166439       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.701µs)
I0410 09:34:14.166484       1 controller.go:28] processed work item
I0410 09:34:14.540997       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.8µs)
I0410 09:34:14.541036       1 controller.go:28] processed work item
I0410 09:34:15.905061       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:15.905092       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:34:16,268][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:34:16,269][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:34:17.263746       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:17.346683       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.101µs)
I0410 09:34:17.346734       1 controller.go:28] processed work item
I0410 09:34:17.741026       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (16.1µs)
I0410 09:34:17.741066       1 controller.go:28] processed work item
I0410 09:34:18.040755       1 logs.go:41] http: panic serving 10.0.16.9:57802: runtime error: invalid memory address or nil pointer dereference
goroutine 2592 [running]:
net/http.(*conn).serve.func1(0xc4201fdd60)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e6060, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42075a600, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205efb40, 0x1b57140, 0xc4205e6060, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420728200, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d88c0, 0xc420257800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d88c0, 0xc420257800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d88c0, 0xc420257800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fdd60, 0x1b570c0, 0xc420624680)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:34:18.888723       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:18.888756       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:34:19,270][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:34:19,272][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:20.946121       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (40.801µs)
I0410 09:34:20.946171       1 controller.go:28] processed work item
I0410 09:34:21.234380       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.9µs)
I0410 09:34:21.234456       1 controller.go:28] processed work item
I0410 09:34:21.541103       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.7µs)
I0410 09:34:21.541153       1 controller.go:28] processed work item
I0410 09:34:21.746567       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (25.101µs)
I0410 09:34:21.746613       1 controller.go:28] processed work item
E0410 09:34:22.270465       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:34:22,274][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:34:22,274][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:22.294989       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22.601µs)
I0410 09:34:22.295028       1 controller.go:28] processed work item
I0410 09:34:22.295056       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (16.6µs)
I0410 09:34:22.295069       1 controller.go:28] processed work item
I0410 09:34:22.341520       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.6µs)
I0410 09:34:22.341561       1 controller.go:28] processed work item
I0410 09:34:23.077026       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:23.077069       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:24.546516       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.7µs)
I0410 09:34:24.546576       1 controller.go:28] processed work item
[2018-04-10T09:34:25,275][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:34:27.277279       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:27.348630       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:27.348695       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:27.746337       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.7µs)
I0410 09:34:27.746382       1 controller.go:28] processed work item
I0410 09:34:28.040935       1 logs.go:41] http: panic serving 10.0.16.9:57966: runtime error: invalid memory address or nil pointer dereference
goroutine 2630 [running]:
net/http.(*conn).serve.func1(0xc420737ae0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205acd80, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bb1a0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420780b40, 0x1b57140, 0xc4205acd80, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328080, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4201f8380, 0xc420700300)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4201f8380, 0xc420700300)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4201f8380, 0xc420700300)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420737ae0, 0x1b570c0, 0xc42060fa80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:34:28.340885       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (13.901µs)
I0410 09:34:28.340919       1 controller.go:28] processed work item
[2018-04-10T09:34:30,276][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:34:30.549885       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:30.549925       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:31.239779       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.501µs)
I0410 09:34:31.239838       1 controller.go:28] processed work item
I0410 09:34:31.546500       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (36.001µs)
I0410 09:34:31.546560       1 controller.go:28] processed work item
I0410 09:34:32.140967       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.1µs)
I0410 09:34:32.141030       1 controller.go:28] processed work item
E0410 09:34:32.283623       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:32.300511       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.101µs)
I0410 09:34:32.300570       1 controller.go:28] processed work item
I0410 09:34:32.300594       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11.201µs)
I0410 09:34:32.300608       1 controller.go:28] processed work item
I0410 09:34:32.347032       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.5µs)
I0410 09:34:32.347092       1 controller.go:28] processed work item
I0410 09:34:32.941101       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.3µs)
I0410 09:34:32.941153       1 controller.go:28] processed work item
I0410 09:34:33.141080       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.5µs)
I0410 09:34:33.141120       1 controller.go:28] processed work item
I0410 09:34:33.247375       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:33.247435       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:34:33,276][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:34:34.551878       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.9µs)
I0410 09:34:34.551969       1 controller.go:28] processed work item
E0410 09:34:37.290207       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:37.596922       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:37.596955       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:38.040741       1 logs.go:41] http: panic serving 10.0.16.9:58128: runtime error: invalid memory address or nil pointer dereference
goroutine 2662 [running]:
net/http.(*conn).serve.func1(0xc4206fe5a0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e6ea0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420338e70, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420780b40, 0x1b57140, 0xc4205e6ea0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420729000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d89a0, 0xc42078d000)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d89a0, 0xc42078d000)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d89a0, 0xc42078d000)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206fe5a0, 0x1b570c0, 0xc420625300)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:34:38,277][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:34:38.346605       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.001µs)
I0410 09:34:38.346655       1 controller.go:28] processed work item
I0410 09:34:38.540932       1 reflector.go:428] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: Watch close - *v1alpha1.Pilot total 168 items received
I0410 09:34:38.947352       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.401µs)
I0410 09:34:38.947391       1 controller.go:28] processed work item
I0410 09:34:40.541615       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:34:40.687696       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:40.687740       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:41.245251       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.1µs)
I0410 09:34:41.245312       1 controller.go:28] processed work item
[2018-04-10T09:34:41,278][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:34:42.146372       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.4µs)
I0410 09:34:42.146430       1 controller.go:28] processed work item
E0410 09:34:42.296640       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:42.305873       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (21.9µs)
I0410 09:34:42.305922       1 controller.go:28] processed work item
I0410 09:34:42.305944       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.301µs)
I0410 09:34:42.305958       1 controller.go:28] processed work item
I0410 09:34:42.749970       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (31.9µs)
I0410 09:34:42.750020       1 controller.go:28] processed work item
I0410 09:34:43.146505       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.1µs)
I0410 09:34:43.146545       1 controller.go:28] processed work item
I0410 09:34:43.347284       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27µs)
I0410 09:34:43.347324       1 controller.go:28] processed work item
I0410 09:34:43.449611       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:43.449658       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:44.557294       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.701µs)
I0410 09:34:44.557351       1 controller.go:28] processed work item
[2018-04-10T09:34:45,290][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:45.747152       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.101µs)
I0410 09:34:45.747204       1 controller.go:28] processed work item
E0410 09:34:47.304372       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:47.739353       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:47.739391       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:48.040632       1 logs.go:41] http: panic serving 10.0.16.9:58300: runtime error: invalid memory address or nil pointer dereference
goroutine 2680 [running]:
net/http.(*conn).serve.func1(0xc4200a15e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420708240, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4206325a0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420041b40, 0x1b57140, 0xc420708240, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420144280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc42009c1c0, 0xc4202f6600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc42009c1c0, 0xc4202f6600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc42009c1c0, 0xc4202f6600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4200a15e0, 0x1b570c0, 0xc42060e440)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:34:48,291][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:34:48,291][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:48.952691       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.201µs)
I0410 09:34:48.953007       1 controller.go:28] processed work item
I0410 09:34:49.547414       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (36.501µs)
I0410 09:34:49.547596       1 controller.go:28] processed work item
I0410 09:34:51.250631       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.3µs)
I0410 09:34:51.250672       1 controller.go:28] processed work item
[2018-04-10T09:34:51,293][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:34:51,293][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:51.547788       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:51.547861       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:34:52.310578       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:52.311256       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.5µs)
I0410 09:34:52.311291       1 controller.go:28] processed work item
I0410 09:34:52.311313       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.101µs)
I0410 09:34:52.311326       1 controller.go:28] processed work item
I0410 09:34:52.755349       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.001µs)
I0410 09:34:52.755405       1 controller.go:28] processed work item
I0410 09:34:53.347238       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.5µs)
I0410 09:34:53.347276       1 controller.go:28] processed work item
I0410 09:34:53.352660       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (14.8µs)
I0410 09:34:53.352701       1 controller.go:28] processed work item
I0410 09:34:53.947770       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.201µs)
I0410 09:34:53.947809       1 controller.go:28] processed work item
[2018-04-10T09:34:54,294][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:34:54,295][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:34:54.838731       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:54.838775       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:55.752512       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.801µs)
I0410 09:34:55.752553       1 controller.go:28] processed work item
[2018-04-10T09:34:57,296][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:34:57.318133       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:34:58.041073       1 logs.go:41] http: panic serving 10.0.16.9:58460: runtime error: invalid memory address or nil pointer dereference
goroutine 2708 [running]:
net/http.(*conn).serve.func1(0xc420171ea0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420456ae0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42062eed0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420041b40, 0x1b57140, 0xc420456ae0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dcc00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8460, 0xc420256a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8460, 0xc420256a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8460, 0xc420256a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420171ea0, 0x1b570c0, 0xc420260a80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:34:58.452720       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:34:58.452777       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:34:59.552868       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (24.901µs)
I0410 09:34:59.552915       1 controller.go:28] processed work item
I0410 09:34:59.947361       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (28.901µs)
I0410 09:34:59.947423       1 controller.go:28] processed work item
I0410 09:35:01.256105       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (31.901µs)
I0410 09:35:01.256156       1 controller.go:28] processed work item
[2018-04-10T09:35:02,296][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:35:02.316710       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22µs)
I0410 09:35:02.316753       1 controller.go:28] processed work item
I0410 09:35:02.316775       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10µs)
I0410 09:35:02.316788       1 controller.go:28] processed work item
E0410 09:35:02.324909       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:02.534678       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:02.534711       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:03.352607       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (48.101µs)
I0410 09:35:03.352658       1 controller.go:28] processed work item
I0410 09:35:03.953186       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (41.301µs)
I0410 09:35:03.953257       1 controller.go:28] processed work item
I0410 09:35:03.955026       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.9µs)
I0410 09:35:03.955056       1 controller.go:28] processed work item
I0410 09:35:04.347068       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (22.901µs)
I0410 09:35:04.347120       1 controller.go:28] processed work item
I0410 09:35:04.547007       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.8µs)
I0410 09:35:04.547045       1 controller.go:28] processed work item
[2018-04-10T09:35:05,297][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:35:05.669768       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:05.669823       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:05.757894       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.601µs)
I0410 09:35:05.757934       1 controller.go:28] processed work item
I0410 09:35:06.747065       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.501µs)
I0410 09:35:06.747113       1 controller.go:28] processed work item
E0410 09:35:07.334294       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:08.040769       1 logs.go:41] http: panic serving 10.0.16.9:58626: runtime error: invalid memory address or nil pointer dereference
goroutine 2701 [running]:
net/http.(*conn).serve.func1(0xc420436780)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4201df740, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420299b00, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42052bb40, 0x1b57140, 0xc4201df740, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202e6580, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc420586000, 0xc420819900)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc420586000, 0xc420819900)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc420586000, 0xc420819900)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436780, 0x1b570c0, 0xc4204c7540)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:35:09.062612       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:09.062647       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:09.952765       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.9µs)
I0410 09:35:09.952818       1 controller.go:28] processed work item
[2018-04-10T09:35:10,298][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:35:10.541972       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:35:10.547028       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (14.8µs)
I0410 09:35:10.547059       1 controller.go:28] processed work item
I0410 09:35:11.261573       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (26.601µs)
I0410 09:35:11.261653       1 controller.go:28] processed work item
I0410 09:35:12.223474       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:12.223516       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:12.322133       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (22µs)
I0410 09:35:12.322182       1 controller.go:28] processed work item
I0410 09:35:12.322203       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.101µs)
I0410 09:35:12.322275       1 controller.go:28] processed work item
E0410 09:35:12.340730       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:35:13,298][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:35:13.960494       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (21.701µs)
I0410 09:35:13.960547       1 controller.go:28] processed work item
I0410 09:35:14.347090       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.6µs)
I0410 09:35:14.347135       1 controller.go:28] processed work item
I0410 09:35:14.552469       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.401µs)
I0410 09:35:14.552522       1 controller.go:28] processed work item
I0410 09:35:14.748325       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (29.5µs)
I0410 09:35:14.748385       1 controller.go:28] processed work item
I0410 09:35:15.549684       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:15.549712       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:16.752457       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.801µs)
I0410 09:35:16.752490       1 controller.go:28] processed work item
[2018-04-10T09:35:17,309][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:17.347441       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (16.7µs)
I0410 09:35:17.347470       1 controller.go:28] processed work item
E0410 09:35:17.347704       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:18.041398       1 logs.go:41] http: panic serving 10.0.16.9:58792: runtime error: invalid memory address or nil pointer dereference
goroutine 2759 [running]:
net/http.(*conn).serve.func1(0xc42081e780)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e2ae0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42016c540, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077eb40, 0x1b57140, 0xc4204e2ae0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043ab00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8540, 0xc420700100)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8540, 0xc420700100)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8540, 0xc420700100)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42081e780, 0x1b570c0, 0xc42041a840)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:35:19.835898       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:19.835933       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:35:20,310][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:35:20,311][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:20.552472       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.601µs)
I0410 09:35:20.552541       1 controller.go:28] processed work item
I0410 09:35:21.147280       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.1µs)
I0410 09:35:21.147358       1 controller.go:28] processed work item
I0410 09:35:21.266977       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (20.101µs)
I0410 09:35:21.267055       1 controller.go:28] processed work item
I0410 09:35:22.327628       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (30.501µs)
I0410 09:35:22.327665       1 controller.go:28] processed work item
I0410 09:35:22.327686       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.701µs)
I0410 09:35:22.327700       1 controller.go:28] processed work item
E0410 09:35:22.353803       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:23.065305       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:23.065355       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:35:23,312][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:35:23,312][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:24.352492       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.901µs)
I0410 09:35:24.352530       1 controller.go:28] processed work item
I0410 09:35:24.747150       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.5µs)
I0410 09:35:24.747205       1 controller.go:28] processed work item
I0410 09:35:24.753674       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.9µs)
I0410 09:35:24.753721       1 controller.go:28] processed work item
I0410 09:35:24.947092       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.101µs)
I0410 09:35:24.947138       1 controller.go:28] processed work item
I0410 09:35:25.147044       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.1µs)
I0410 09:35:25.147084       1 controller.go:28] processed work item
[2018-04-10T09:35:26,314][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:35:26,314][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:26.851801       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:26.851836       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:27.352807       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (40.901µs)
I0410 09:35:27.352859       1 controller.go:28] processed work item
E0410 09:35:27.360305       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:28.040984       1 logs.go:41] http: panic serving 10.0.16.9:58954: runtime error: invalid memory address or nil pointer dereference
goroutine 2750 [running]:
net/http.(*conn).serve.func1(0xc420436f00)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420273320, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bbe30, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420526b40, 0x1b57140, 0xc420273320, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420321000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4205861c0, 0xc42078d600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4205861c0, 0xc42078d600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4205861c0, 0xc42078d600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420436f00, 0x1b570c0, 0xc42049ca80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:35:29,315][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:35:30.047602       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:30.047635       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:31.152700       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.7µs)
I0410 09:35:31.152866       1 controller.go:28] processed work item
I0410 09:35:31.272374       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (55.201µs)
I0410 09:35:31.272423       1 controller.go:28] processed work item
I0410 09:35:31.547006       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.5µs)
I0410 09:35:31.547054       1 controller.go:28] processed work item
I0410 09:35:32.210248       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:32.210280       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:32.332991       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (23.5µs)
I0410 09:35:32.333185       1 controller.go:28] processed work item
I0410 09:35:32.333282       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (13.101µs)
I0410 09:35:32.333375       1 controller.go:28] processed work item
E0410 09:35:32.366961       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:35:34,316][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:35:34.845379       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:34.845435       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:34.952495       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (30.801µs)
I0410 09:35:34.952535       1 controller.go:28] processed work item
I0410 09:35:35.152476       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (20.4µs)
I0410 09:35:35.152539       1 controller.go:28] processed work item
I0410 09:35:35.347034       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.8µs)
I0410 09:35:35.347067       1 controller.go:28] processed work item
I0410 09:35:35.547049       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.1µs)
I0410 09:35:35.547105       1 controller.go:28] processed work item
I0410 09:35:35.948080       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (33.101µs)
I0410 09:35:35.948172       1 controller.go:28] processed work item
[2018-04-10T09:35:37,317][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:35:37.358251       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (51.101µs)
I0410 09:35:37.358486       1 controller.go:28] processed work item
E0410 09:35:37.373273       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:38.040704       1 logs.go:41] http: panic serving 10.0.16.9:59126: runtime error: invalid memory address or nil pointer dereference
goroutine 2795 [running]:
net/http.(*conn).serve.func1(0xc42081ef00)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e3ec0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202587e0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077eb40, 0x1b57140, 0xc4204e3ec0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043bf00, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8700, 0xc420701a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8700, 0xc420701a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8700, 0xc420701a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42081ef00, 0x1b570c0, 0xc42041b7c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:35:38.347042       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.1µs)
I0410 09:35:38.347082       1 controller.go:28] processed work item
I0410 09:35:39.084569       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:39.084603       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:40.542314       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:35:41.277822       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (21.801µs)
I0410 09:35:41.277868       1 controller.go:28] processed work item
I0410 09:35:41.552400       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.2µs)
I0410 09:35:41.552448       1 controller.go:28] processed work item
I0410 09:35:41.947116       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (26.5µs)
I0410 09:35:41.947164       1 controller.go:28] processed work item
I0410 09:35:41.984497       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:41.984527       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:35:42,317][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:35:42.338468       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (21.901µs)
I0410 09:35:42.338517       1 controller.go:28] processed work item
I0410 09:35:42.338547       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (20.6µs)
I0410 09:35:42.338560       1 controller.go:28] processed work item
E0410 09:35:42.379821       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:44.971897       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:44.971948       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:35:45,318][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:35:45.352473       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.101µs)
I0410 09:35:45.352517       1 controller.go:28] processed work item
I0410 09:35:45.947431       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.1µs)
I0410 09:35:45.947473       1 controller.go:28] processed work item
I0410 09:35:45.954697       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (14.901µs)
I0410 09:35:45.954743       1 controller.go:28] processed work item
I0410 09:35:46.147170       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.1µs)
I0410 09:35:46.147261       1 controller.go:28] processed work item
I0410 09:35:46.346933       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.7µs)
I0410 09:35:46.346974       1 controller.go:28] processed work item
I0410 09:35:46.747051       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (37.201µs)
I0410 09:35:46.747101       1 controller.go:28] processed work item
E0410 09:35:47.386294       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:47.973773       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:47.973815       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:48.040716       1 logs.go:41] http: panic serving 10.0.16.9:59276: runtime error: invalid memory address or nil pointer dereference
goroutine 2828 [running]:
net/http.(*conn).serve.func1(0xc420630b40)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205ae120, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420290a20, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420781b40, 0x1b57140, 0xc4205ae120, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4280, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4206fa2a0, 0xc4204b6400)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4206fa2a0, 0xc4204b6400)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4206fa2a0, 0xc4204b6400)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420630b40, 0x1b570c0, 0xc420609280)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:35:48.352420       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.701µs)
I0410 09:35:48.352474       1 controller.go:28] processed work item
[2018-04-10T09:35:49,337][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:50.212174       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:50.212207       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:51.283282       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.801µs)
I0410 09:35:51.283329       1 controller.go:28] processed work item
I0410 09:35:51.952600       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.4µs)
I0410 09:35:51.952654       1 controller.go:28] processed work item
[2018-04-10T09:35:52,338][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:35:52,338][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:52.343838       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (20.901µs)
I0410 09:35:52.343898       1 controller.go:28] processed work item
I0410 09:35:52.343934       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.8µs)
I0410 09:35:52.343949       1 controller.go:28] processed work item
E0410 09:35:52.392952       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:52.547150       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.7µs)
I0410 09:35:52.547335       1 controller.go:28] processed work item
I0410 09:35:54.380111       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:54.380143       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:35:55,339][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:35:55,340][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:56.152556       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.6µs)
I0410 09:35:56.152689       1 controller.go:28] processed work item
I0410 09:35:56.395176       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:56.395235       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:56.752475       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.201µs)
I0410 09:35:56.752534       1 controller.go:28] processed work item
I0410 09:35:56.774019       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.601µs)
I0410 09:35:56.774054       1 controller.go:28] processed work item
I0410 09:35:57.347039       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.101µs)
I0410 09:35:57.347089       1 controller.go:28] processed work item
E0410 09:35:57.399268       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:35:58.040718       1 logs.go:41] http: panic serving 10.0.16.9:59450: runtime error: invalid memory address or nil pointer dereference
goroutine 2842 [running]:
net/http.(*conn).serve.func1(0xc42081f5e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc42074c3c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fc7b0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077eb40, 0x1b57140, 0xc42074c3c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420318e80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d87e0, 0xc4201bf300)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d87e0, 0xc4201bf300)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d87e0, 0xc4201bf300)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc42081f5e0, 0x1b570c0, 0xc42008c400)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:35:58,341][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:35:58,342][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:35:58.357852       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (21.901µs)
I0410 09:35:58.357906       1 controller.go:28] processed work item
I0410 09:35:59.056603       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:35:59.056634       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:35:59.347156       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (29.6µs)
I0410 09:35:59.347207       1 controller.go:28] processed work item
I0410 09:36:01.288738       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (24.701µs)
I0410 09:36:01.288780       1 controller.go:28] processed work item
I0410 09:36:01.323315       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:01.323365       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:36:01,343][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:36:02.349233       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.101µs)
I0410 09:36:02.349284       1 controller.go:28] processed work item
I0410 09:36:02.349306       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (10.9µs)
I0410 09:36:02.349318       1 controller.go:28] processed work item
E0410 09:36:02.405477       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:02.552599       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.4µs)
I0410 09:36:02.552637       1 controller.go:28] processed work item
I0410 09:36:03.147252       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (45.201µs)
I0410 09:36:03.147297       1 controller.go:28] processed work item
I0410 09:36:05.381130       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:05.381157       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:36:06,343][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:36:06.779548       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (34.901µs)
I0410 09:36:06.779592       1 controller.go:28] processed work item
I0410 09:36:07.347302       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.4µs)
I0410 09:36:07.347351       1 controller.go:28] processed work item
I0410 09:36:07.352556       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (50.101µs)
I0410 09:36:07.352587       1 controller.go:28] processed work item
E0410 09:36:07.412046       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:07.547099       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17.701µs)
I0410 09:36:07.547196       1 controller.go:28] processed work item
I0410 09:36:07.947358       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.901µs)
I0410 09:36:07.947395       1 controller.go:28] processed work item
I0410 09:36:08.004164       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:08.004195       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:08.040740       1 logs.go:41] http: panic serving 10.0.16.9:59618: runtime error: invalid memory address or nil pointer dereference
goroutine 2871 [running]:
net/http.(*conn).serve.func1(0xc420437900)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420515980, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202c75c0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420781b40, 0x1b57140, 0xc420515980, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4202a6e80, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc420586380, 0xc420379b00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc420586380, 0xc420379b00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc420586380, 0xc420379b00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437900, 0x1b570c0, 0xc42058c5c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:36:09,344][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:36:09.352506       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.9µs)
I0410 09:36:09.352566       1 controller.go:28] processed work item
I0410 09:36:10.147119       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.401µs)
I0410 09:36:10.147193       1 controller.go:28] processed work item
I0410 09:36:10.542644       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:36:11.294073       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22µs)
I0410 09:36:11.294114       1 controller.go:28] processed work item
I0410 09:36:12.354584       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.901µs)
I0410 09:36:12.354615       1 controller.go:28] processed work item
I0410 09:36:12.354636       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.1µs)
I0410 09:36:12.354648       1 controller.go:28] processed work item
I0410 09:36:12.382954       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:12.382982       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:36:12.418280       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:13.152699       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (25.301µs)
I0410 09:36:13.152741       1 controller.go:28] processed work item
I0410 09:36:13.747194       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.5µs)
I0410 09:36:13.747243       1 controller.go:28] processed work item
[2018-04-10T09:36:14,345][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:36:16.610995       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:16.611038       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:36:17,345][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:36:17.425824       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:17.552522       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (24.5µs)
I0410 09:36:17.552567       1 controller.go:28] processed work item
I0410 09:36:17.947145       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.7µs)
I0410 09:36:17.947180       1 controller.go:28] processed work item
I0410 09:36:17.952735       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (10.201µs)
I0410 09:36:17.952761       1 controller.go:28] processed work item
I0410 09:36:18.040570       1 logs.go:41] http: panic serving 10.0.16.9:59790: runtime error: invalid memory address or nil pointer dereference
goroutine 2879 [running]:
net/http.(*conn).serve.func1(0xc420437cc0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e63c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420338840, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077eb40, 0x1b57140, 0xc4205e63c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420328080, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc420586460, 0xc4206d0600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc420586460, 0xc4206d0600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc420586460, 0xc4206d0600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420437cc0, 0x1b570c0, 0xc42058cd40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:36:18.547713       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.401µs)
I0410 09:36:18.547828       1 controller.go:28] processed work item
I0410 09:36:19.026435       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:19.026469       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:20.152604       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (25.3µs)
I0410 09:36:20.152660       1 controller.go:28] processed work item
I0410 09:36:21.299484       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (22.301µs)
I0410 09:36:21.299528       1 controller.go:28] processed work item
[2018-04-10T09:36:21,356][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:36:21.761695       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:21.761724       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:22.360013       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.201µs)
I0410 09:36:22.360057       1 controller.go:28] processed work item
I0410 09:36:22.360085       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (16.201µs)
I0410 09:36:22.360111       1 controller.go:28] processed work item
E0410 09:36:22.432519       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:23.752630       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (28.501µs)
I0410 09:36:23.752680       1 controller.go:28] processed work item
I0410 09:36:24.147549       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (20.901µs)
I0410 09:36:24.147586       1 controller.go:28] processed work item
I0410 09:36:24.347250       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (18.001µs)
I0410 09:36:24.347302       1 controller.go:28] processed work item
[2018-04-10T09:36:24,357][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:36:24,358][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:36:25.050609       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:25.050650       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:36:27,359][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:36:27,359][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:36:27.439934       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:27.479540       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:27.479558       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:27.952691       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.201µs)
I0410 09:36:27.952732       1 controller.go:28] processed work item
I0410 09:36:28.041044       1 logs.go:41] http: panic serving 10.0.16.9:59950: runtime error: invalid memory address or nil pointer dereference
goroutine 2926 [running]:
net/http.(*conn).serve.func1(0xc4206be280)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205e6e40, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42083e6f0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077eb40, 0x1b57140, 0xc4205e6e40, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420329000, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc420586540, 0xc4206d1a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc420586540, 0xc4206d1a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc420586540, 0xc4206d1a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206be280, 0x1b570c0, 0xc42058d800)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:36:28.147170       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.701µs)
I0410 09:36:28.147283       1 controller.go:28] processed work item
I0410 09:36:28.553148       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.901µs)
I0410 09:36:28.553183       1 controller.go:28] processed work item
I0410 09:36:29.147310       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (28.401µs)
I0410 09:36:29.147362       1 controller.go:28] processed work item
I0410 09:36:30.158111       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.901µs)
I0410 09:36:30.158192       1 controller.go:28] processed work item
[2018-04-10T09:36:30,360][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:36:30,360][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:36:31.304832       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (25.1µs)
I0410 09:36:31.304887       1 controller.go:28] processed work item
I0410 09:36:31.438310       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:31.438354       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:32.365384       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (35.101µs)
I0410 09:36:32.365435       1 controller.go:28] processed work item
I0410 09:36:32.365458       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (11.7µs)
I0410 09:36:32.365510       1 controller.go:28] processed work item
E0410 09:36:32.446462       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
[2018-04-10T09:36:33,361][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:36:34.352629       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (34.401µs)
I0410 09:36:34.352661       1 controller.go:28] processed work item
I0410 09:36:34.747042       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (16.6µs)
I0410 09:36:34.747080       1 controller.go:28] processed work item
I0410 09:36:34.947157       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (37.1µs)
I0410 09:36:34.947200       1 controller.go:28] processed work item
I0410 09:36:35.135400       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:35.135444       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:36:37.452330       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:37.758331       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:37.758364       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:38.042005       1 logs.go:41] http: panic serving 10.0.16.9:60120: runtime error: invalid memory address or nil pointer dereference
goroutine 2911 [running]:
net/http.(*conn).serve.func1(0xc420171d60)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4205152c0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420339470, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc4205152c0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420319080, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8380, 0xc420256900)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8380, 0xc420256900)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8380, 0xc420256900)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420171d60, 0x1b570c0, 0xc42049c540)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:36:38.152568       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (22.2µs)
I0410 09:36:38.152611       1 controller.go:28] processed work item
[2018-04-10T09:36:38,362][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:36:38.747197       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.1µs)
I0410 09:36:38.747252       1 controller.go:28] processed work item
I0410 09:36:39.152710       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.101µs)
I0410 09:36:39.152763       1 controller.go:28] processed work item
I0410 09:36:39.747167       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (17.801µs)
I0410 09:36:39.747235       1 controller.go:28] processed work item
I0410 09:36:40.163521       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (27.101µs)
I0410 09:36:40.163570       1 controller.go:28] processed work item
I0410 09:36:40.362974       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:40.363060       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:40.542995       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:36:41.310294       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (78.002µs)
I0410 09:36:41.310430       1 controller.go:28] processed work item
[2018-04-10T09:36:41,363][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:36:41.547270       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (41.601µs)
I0410 09:36:41.547311       1 controller.go:28] processed work item
I0410 09:36:42.370782       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.101µs)
I0410 09:36:42.370843       1 controller.go:28] processed work item
I0410 09:36:42.370864       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.901µs)
I0410 09:36:42.370876       1 controller.go:28] processed work item
E0410 09:36:42.459395       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:43.172166       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:43.172208       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:44.952553       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (34.6µs)
I0410 09:36:44.952653       1 controller.go:28] processed work item
I0410 09:36:45.547234       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (29.601µs)
I0410 09:36:45.547268       1 controller.go:28] processed work item
[2018-04-10T09:36:46,363][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:36:46.979957       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:46.980008       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:36:47.465465       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:48.040759       1 logs.go:41] http: panic serving 10.0.16.9:60274: runtime error: invalid memory address or nil pointer dereference
goroutine 2954 [running]:
net/http.(*conn).serve.func1(0xc4201fc140)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc4204e30e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4202fcea0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420041b40, 0x1b57140, 0xc4204e30e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4203e9780, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4205860e0, 0xc4202f6c00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4205860e0, 0xc4202f6c00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4205860e0, 0xc4202f6c00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fc140, 0x1b570c0, 0xc420345540)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:36:48.752661       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (78.301µs)
I0410 09:36:48.752712       1 controller.go:28] processed work item
I0410 09:36:48.996148       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:48.996175       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:49.347459       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.7µs)
I0410 09:36:49.347494       1 controller.go:28] processed work item
[2018-04-10T09:36:49,364][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:36:49.752590       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (55.101µs)
I0410 09:36:49.752689       1 controller.go:28] processed work item
I0410 09:36:50.147197       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.101µs)
I0410 09:36:50.147332       1 controller.go:28] processed work item
I0410 09:36:51.315717       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (27µs)
I0410 09:36:51.315810       1 controller.go:28] processed work item
I0410 09:36:51.552858       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22µs)
I0410 09:36:51.552901       1 controller.go:28] processed work item
I0410 09:36:52.376159       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.301µs)
I0410 09:36:52.376202       1 controller.go:28] processed work item
I0410 09:36:52.376303       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11µs)
I0410 09:36:52.376320       1 controller.go:28] processed work item
E0410 09:36:52.472123       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:53.022418       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:53.022449       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:36:53,376][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:36:55.552594       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (22.601µs)
I0410 09:36:55.552633       1 controller.go:28] processed work item
I0410 09:36:55.580870       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:55.580930       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:56.147269       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.1µs)
I0410 09:36:56.147308       1 controller.go:28] processed work item
[2018-04-10T09:36:56,377][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:36:56,378][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:36:57.478546       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:36:57.617846       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:36:57.617879       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:36:58.040896       1 logs.go:41] http: panic serving 10.0.16.9:60454: runtime error: invalid memory address or nil pointer dereference
goroutine 3011 [running]:
net/http.(*conn).serve.func1(0xc4206be8c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426540, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021e120, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420426540, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043a380, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8540, 0xc420240a00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8540, 0xc420240a00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8540, 0xc420240a00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206be8c0, 0x1b570c0, 0xc42049ddc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:36:59.352862       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (28.501µs)
I0410 09:36:59.352910       1 controller.go:28] processed work item
[2018-04-10T09:36:59,379][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:36:59,379][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:36:59.947429       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (17µs)
I0410 09:36:59.947467       1 controller.go:28] processed work item
I0410 09:37:00.152579       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (23.601µs)
I0410 09:37:00.152614       1 controller.go:28] processed work item
I0410 09:37:00.747247       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.301µs)
I0410 09:37:00.747295       1 controller.go:28] processed work item
I0410 09:37:01.321112       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.501µs)
I0410 09:37:01.321153       1 controller.go:28] processed work item
I0410 09:37:01.558325       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.601µs)
I0410 09:37:01.558376       1 controller.go:28] processed work item
I0410 09:37:01.902161       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:01.902237       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:37:02,380][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:37:02.381556       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.101µs)
I0410 09:37:02.381638       1 controller.go:28] processed work item
I0410 09:37:02.381717       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (10.8µs)
I0410 09:37:02.381756       1 controller.go:28] processed work item
[2018-04-10T09:37:02,381][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:37:02.485286       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:02.747166       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (18.6µs)
I0410 09:37:02.747208       1 controller.go:28] processed work item
[2018-04-10T09:37:05,382][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:37:06.065294       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:06.065406       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:06.152724       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.4µs)
I0410 09:37:06.152820       1 controller.go:28] processed work item
I0410 09:37:06.548070       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (31.3µs)
I0410 09:37:06.548109       1 controller.go:28] processed work item
E0410 09:37:07.496098       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:08.040931       1 logs.go:41] http: panic serving 10.0.16.9:60610: runtime error: invalid memory address or nil pointer dereference
goroutine 3022 [running]:
net/http.(*conn).serve.func1(0xc4206bec80)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420426e40, 0x130297b, 0x3, 0x130af27, 0x10, 0xc42021f350, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420426e40, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc42043b080, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4202d8700, 0xc420241800)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4202d8700, 0xc420241800)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4202d8700, 0xc420241800)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4206bec80, 0x1b570c0, 0xc420260fc0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:37:09.952845       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (23.1µs)
I0410 09:37:09.952948       1 controller.go:28] processed work item
I0410 09:37:10.379589       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:10.379631       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:37:10,382][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:37:10.543246       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:37:10.547642       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (16.2µs)
I0410 09:37:10.547672       1 controller.go:28] processed work item
I0410 09:37:10.752609       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (24.801µs)
I0410 09:37:10.752663       1 controller.go:28] processed work item
I0410 09:37:11.326524       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (44.101µs)
I0410 09:37:11.326584       1 controller.go:28] processed work item
I0410 09:37:11.347077       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (12.3µs)
I0410 09:37:11.347130       1 controller.go:28] processed work item
I0410 09:37:12.386919       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (23.6µs)
I0410 09:37:12.386957       1 controller.go:28] processed work item
I0410 09:37:12.387044       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (15.7µs)
I0410 09:37:12.387075       1 controller.go:28] processed work item
I0410 09:37:12.486957       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:12.486989       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:37:12.502518       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:12.752591       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (24.401µs)
I0410 09:37:12.752633       1 controller.go:28] processed work item
[2018-04-10T09:37:13,383][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:37:13.547342       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (15.9µs)
I0410 09:37:13.547428       1 controller.go:28] processed work item
I0410 09:37:16.201924       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:16.201950       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:16.553536       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (23.1µs)
I0410 09:37:16.553577       1 controller.go:28] processed work item
I0410 09:37:16.947468       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19.2µs)
I0410 09:37:16.947510       1 controller.go:28] processed work item
E0410 09:37:17.509320       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:18.041353       1 logs.go:41] http: panic serving 10.0.16.9:60774: runtime error: invalid memory address or nil pointer dereference
goroutine 3059 [running]:
net/http.(*conn).serve.func1(0xc4201fcbe0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420457ce0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420075b60, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420041b40, 0x1b57140, 0xc420457ce0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420274980, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4205862a0, 0xc4201e5100)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4205862a0, 0xc4201e5100)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4205862a0, 0xc4201e5100)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fcbe0, 0x1b570c0, 0xc42008de40)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:37:18.329094       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:18.329172       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:37:18,385][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:37:20.553008       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.2µs)
I0410 09:37:20.553053       1 controller.go:28] processed work item
I0410 09:37:21.148085       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.801µs)
I0410 09:37:21.148120       1 controller.go:28] processed work item
I0410 09:37:21.311831       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:21.311876       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:21.331841       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (18.1µs)
I0410 09:37:21.331903       1 controller.go:28] processed work item
I0410 09:37:21.352465       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (15.001µs)
I0410 09:37:21.352489       1 controller.go:28] processed work item
[2018-04-10T09:37:21,386][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:37:21.947199       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (16.3µs)
I0410 09:37:21.947250       1 controller.go:28] processed work item
I0410 09:37:22.392321       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (22.301µs)
I0410 09:37:22.392372       1 controller.go:28] processed work item
I0410 09:37:22.392393       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.4µs)
I0410 09:37:22.392405       1 controller.go:28] processed work item
E0410 09:37:22.515635       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:23.552729       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.1µs)
I0410 09:37:23.552813       1 controller.go:28] processed work item
I0410 09:37:24.147109       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (17.201µs)
I0410 09:37:24.147147       1 controller.go:28] processed work item
I0410 09:37:24.458332       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:24.458448       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:37:25,386][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:37:26.952950       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (21.201µs)
I0410 09:37:26.952999       1 controller.go:28] processed work item
I0410 09:37:27.296703       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:27.296733       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
E0410 09:37:27.523367       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:27.551329       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (16.901µs)
I0410 09:37:27.551366       1 controller.go:28] processed work item
I0410 09:37:28.040857       1 logs.go:41] http: panic serving 10.0.16.9:60936: runtime error: invalid memory address or nil pointer dereference
goroutine 3074 [running]:
net/http.(*conn).serve.func1(0xc4201fd220)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420709320, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4203bb710, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc4205f0b40, 0x1b57140, 0xc420709320, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc420231580, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc420586460, 0xc42042a600)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc420586460, 0xc42042a600)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc420586460, 0xc42042a600)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4201fd220, 0x1b570c0, 0xc42060eb80)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
[2018-04-10T09:37:28,387][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:37:28,388][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:37:29.400195       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:29.400272       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:31.153550       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (25.6µs)
I0410 09:37:31.153606       1 controller.go:28] processed work item
I0410 09:37:31.337374       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (23.401µs)
I0410 09:37:31.337416       1 controller.go:28] processed work item
[2018-04-10T09:37:31,389][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:37:31,389][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:37:31.747179       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.201µs)
I0410 09:37:31.747246       1 controller.go:28] processed work item
I0410 09:37:31.952519       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.701µs)
I0410 09:37:31.952565       1 controller.go:28] processed work item
I0410 09:37:32.397666       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (51.401µs)
I0410 09:37:32.397708       1 controller.go:28] processed work item
I0410 09:37:32.397727       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (8.8µs)
I0410 09:37:32.397750       1 controller.go:28] processed work item
E0410 09:37:32.529302       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:32.547084       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (14.7µs)
I0410 09:37:32.547126       1 controller.go:28] processed work item
I0410 09:37:32.839105       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:32.839136       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:34.152509       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.9µs)
I0410 09:37:34.152553       1 controller.go:28] processed work item
I0410 09:37:34.347245       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (34.001µs)
I0410 09:37:34.347291       1 controller.go:28] processed work item
[2018-04-10T09:37:34,391][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
[2018-04-10T09:37:34,391][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
I0410 09:37:35.475143       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:35.475173       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
[2018-04-10T09:37:37,392][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:37:37.535790       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:37.556701       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (19µs)
I0410 09:37:37.556744       1 controller.go:28] processed work item
I0410 09:37:38.040715       1 logs.go:41] http: panic serving 10.0.16.9:32874: runtime error: invalid memory address or nil pointer dereference
goroutine 2857 [running]:
net/http.(*conn).serve.func1(0xc4205821e0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc42074c4e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc4207bab10, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077cb40, 0x1b57140, 0xc42074c4e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d4680, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4206fe000, 0xc4201be700)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4206fe000, 0xc4201be700)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc4206fe000, 0xc4201be700)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4205821e0, 0x1b570c0, 0xc4206244c0)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:37:38.347138       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (17.001µs)
I0410 09:37:38.347204       1 controller.go:28] processed work item
I0410 09:37:39.478281       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:39.478325       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:40.543438       1 reflector.go:286] github.com/jetstack/navigator/pkg/client/informers/externalversions/factory.go:73: forcing resync
I0410 09:37:41.342758       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (31.301µs)
I0410 09:37:41.342827       1 controller.go:28] processed work item
I0410 09:37:41.752519       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (32.901µs)
I0410 09:37:41.752559       1 controller.go:28] processed work item
I0410 09:37:42.347148       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (18.901µs)
I0410 09:37:42.347183       1 controller.go:28] processed work item
[2018-04-10T09:37:42,393][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:37:42.403069       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (21.9µs)
I0410 09:37:42.403113       1 controller.go:28] processed work item
I0410 09:37:42.403136       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (11µs)
I0410 09:37:42.403150       1 controller.go:28] processed work item
E0410 09:37:42.542651       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:42.552494       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (62.601µs)
I0410 09:37:42.552527       1 controller.go:28] processed work item
I0410 09:37:43.147331       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (19.301µs)
I0410 09:37:43.147374       1 controller.go:28] processed work item
I0410 09:37:43.788303       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:43.788346       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:44.352665       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (22.601µs)
I0410 09:37:44.352717       1 controller.go:28] processed work item
I0410 09:37:44.747124       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (28.301µs)
I0410 09:37:44.747172       1 controller.go:28] processed work item
[2018-04-10T09:37:45,393][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
E0410 09:37:47.548837       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:48.040776       1 logs.go:41] http: panic serving 10.0.16.9:33046: runtime error: invalid memory address or nil pointer dereference
goroutine 3096 [running]:
net/http.(*conn).serve.func1(0xc4204377c0)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc420738b40, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420290ab0, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc420555b40, 0x1b57140, 0xc420738b40, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4201dd700, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc420586620, 0xc4203b4f00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc420586620, 0xc4203b4f00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net/http.serverHandler.ServeHTTP(0xc4203940d0, 0x1b56440, 0xc420586620, 0xc4203b4f00)
	/usr/local/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc4204377c0, 0x1b570c0, 0xc42058d580)
	/usr/local/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2720 +0x288
I0410 09:37:48.043428       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:48.043519       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:48.352637       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (31.9µs)
I0410 09:37:48.352682       1 controller.go:28] processed work item
I0410 09:37:48.747088       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-2" (15.9µs)
I0410 09:37:48.747147       1 controller.go:28] processed work item
[2018-04-10T09:37:50,394][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] timed out after [5s] resolving host [es-logging-discovery]
I0410 09:37:50.599203       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:50.599266       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:51.348182       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-2" (46.701µs)
I0410 09:37:51.348251       1 controller.go:28] processed work item
I0410 09:37:52.352561       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (38.401µs)
I0410 09:37:52.352614       1 controller.go:28] processed work item
I0410 09:37:52.408533       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-1" (18.701µs)
I0410 09:37:52.408579       1 controller.go:28] processed work item
I0410 09:37:52.408600       1 controller.go:61] Finished syncing pilot "logging/es-logging-master-0" (9.9µs)
I0410 09:37:52.408612       1 controller.go:28] processed work item
E0410 09:37:52.554706       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:53.147357       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-0" (15.601µs)
I0410 09:37:53.147403       1 controller.go:28] processed work item
I0410 09:37:53.152709       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (21.301µs)
I0410 09:37:53.152733       1 controller.go:28] processed work item
I0410 09:37:53.347365       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (27.101µs)
I0410 09:37:53.347402       1 controller.go:28] processed work item
[2018-04-10T09:37:53,395][WARN ][o.e.d.z.ZenDiscovery     ] [es-logging-mixed-3] not enough master nodes discovered during pinging (found [[]], but needed [2]), pinging again
I0410 09:37:53.547308       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-1" (18.3µs)
I0410 09:37:53.547344       1 controller.go:28] processed work item
I0410 09:37:54.332455       1 leaderelection.go:243] lock is held by es-logging-master-1-external-navigator-controller and has not yet expired
I0410 09:37:54.332501       1 leaderelection.go:180] failed to acquire lease logging/elastic-logging-leaderelection
I0410 09:37:54.752504       1 controller.go:61] Finished syncing pilot "logging/es-logging-mixed-4" (23.401µs)
I0410 09:37:54.752649       1 controller.go:28] processed work item
[2018-04-10T09:37:57,408][WARN ][o.e.d.z.UnicastZenPing   ] [es-logging-mixed-3] failed to resolve host [es-logging-discovery]
java.net.UnknownHostException: es-logging-discovery
	at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[?:1.8.0_161]
	at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[?:1.8.0_161]
	at org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:918) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:873) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:703) ~[elasticsearch-6.2.3.jar:6.2.3]
	at org.elasticsearch.discovery.zen.UnicastZenPing.lambda$resolveHostsLists$0(UnicastZenPing.java:213) ~[elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_161]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) [elasticsearch-6.2.3.jar:6.2.3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
E0410 09:37:57.561480       1 pilot.go:63] Error creating elasticsearch api client: health check timeout: no Elasticsearch node available
I0410 09:37:58.040779       1 logs.go:41] http: panic serving 10.0.16.9:33214: runtime error: invalid memory address or nil pointer dereference
goroutine 3114 [running]:
net/http.(*conn).serve.func1(0xc420582780)
	/usr/local/go/src/net/http/server.go:1697 +0xd0
panic(0x118df40, 0x1ba1770)
	/usr/local/go/src/runtime/panic.go:491 +0x283
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*Client).PerformRequest(0x0, 0x1b57140, 0xc42074d3e0, 0x130297b, 0x3, 0x130af27, 0x10, 0xc420419560, 0x0, 0x0, ...)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/client.go:1179 +0xe3
github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic%2ev5.(*ClusterHealthService).Do(0xc42077cb40, 0x1b57140, 0xc42074d3e0, 0x1bd3908, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/vendor/gopkg.in/olivere/elastic.v5/cluster_health.go:182 +0xe1
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).localNodeHealth(0xc4200844d0, 0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:30 +0x176
github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.(*Pilot).(github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5.localNodeHealth)-fm(0xc4204d5600, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/elasticsearch/v5/probes.go:14 +0x2a
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.CombineChecks.func1(0x0, 0x0)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/check.go:10 +0x60
github.com/jetstack/navigator/pkg/pilot/genericpilot/probe.(*Listener).Listen.func1(0x1b56440, 0xc4206fe0e0, 0xc4201bfc00)
	/go/src/github.com/jetstack/navigator/pkg/pilot/genericpilot/probe/listen.go:20 +0x41
net/http.HandlerFunc.ServeHTTP(0xc42036e140, 0x1b56440, 0xc4206fe0e0, 0xc4201bfc00)
	/usr/local/go/src/net/http/server.go:1918 +0x44
net