ElasticSearch issues

es541_cluster_allocation_explain

# curl -XGET localhost:9200/_cluster/allocation/explain?pretty
{
  "index" : "filebeat-2017.09.08",
  "shard" : 0,
  "primary" : true,
  "current_state" : "unassigned",
  "unassigned_info" : {
    "reason" : "CLUSTER_RECOVERED",
    "at" : "2018-10-12T19:09:07.740Z",
    "last_allocation_status" : "no_valid_shard_copy"
  },
  "can_allocate" : "no_valid_shard_copy",
  "allocate_explanation" : "cannot allocate because all found copies of the shard are either stale or corrupt",
  "node_allocation_decisions" : [
    {
      "node_id" : "5awoLm1yQ0yOkBdLy3FTzQ",
      "node_name" : "5awoLm1",
      "transport_address" : "172.18.0.2:9300",
      "node_decision" : "no",
      "store" : {
        "in_sync" : true,
        "allocation_id" : "l9SRskjDRDeq0PcYbXrKvA",
        "store_exception" : {
          "type" : "index_format_too_new_exception",
          "reason" : "Format version is not supported (resource BufferedChecksumIndexInput(SimpleFSIndexInput(path=\"/var/lib/elasticsearch/nodes/0/indices/xy3ktob4SN-CJoy05XkxWQ/0/index/segments_30\"))): 9 (needs to be between 4 and 6)"
        }
      }
    }
  ]
}

es541_ingest_pipelines

# curl 'localhost:9200/_ingest/pipeline?pretty=1'
{
  "xpack_monitoring_2" : {
    "description" : "This pipeline upgrades documents from the older version of the Monitoring API to the newer version (6) by fixing breaking changes in those older documents before they are indexed from the older version (2).",
    "version" : 6040099,
    "processors" : [
      {
        "script" : {
          "source" : "boolean legacyIndex = ctx._index == '.monitoring-data-2';if (legacyIndex || ctx._index.startsWith('.monitoring-es-2')) {if (ctx._type == 'cluster_info') {ctx._type = 'cluster_stats';ctx._id = null;} else if (legacyIndex || ctx._type == 'cluster_stats' || ctx._type == 'node') {String index = ctx._index;Object clusterUuid = ctx.cluster_uuid;Object timestamp = ctx.timestamp;ctx.clear();ctx._id = 'xpack_monitoring_2_drop_bucket';ctx._index = index;ctx._type = 'legacy_data';ctx.timestamp = timestamp;ctx.cluster_uuid = clusterUuid;}if (legacyIndex) {ctx._index = '<.monitoring-es-6-{now}>';}}"
        }
      },
      {
        "rename" : {
          "field" : "_type",
          "target_field" : "type"
        }
      },
      {
        "set" : {
          "field" : "_type",
          "value" : "doc"
        }
      },
      {
        "gsub" : {
          "field" : "_index",
          "pattern" : "(.monitoring-\\w+-)2(-.+)",
          "replacement" : "$16$2"
        }
      }
    ]
  },
  "xpack_monitoring_6" : {
    "description" : "This is a placeholder pipeline for Monitoring API version 6 so that future versions may fix breaking changes.",
    "version" : 6040099,
    "processors" : [ ]
  }
}

es541_startup.health

# curl localhost:9200/_cluster/health?pretty
{
  "cluster_name" : "elasticsearch",
  "status" : "red",
  "timed_out" : false,
  "number_of_nodes" : 1,
  "number_of_data_nodes" : 1,
  "active_primary_shards" : 0,
  "active_shards" : 0,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 1113,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 0.0
}

es_541_startup.error

[2018-10-12T18:11:33,207][INFO ][o.e.n.Node               ] [] initializing ...
[2018-10-12T18:11:33,261][INFO ][o.e.e.NodeEnvironment    ] [5awoLm1] using [1] data paths, mounts [[/var/lib/elasticsearch (/dev/nvme1n1)]], net usable_space [229.1gb], net total_space [1.4tb], spins? [possibly], types [ext4]
[2018-10-12T18:11:33,262][INFO ][o.e.e.NodeEnvironment    ] [5awoLm1] heap size [21.9gb], compressed ordinary object pointers [true]
[2018-10-12T18:11:34,498][INFO ][o.e.n.Node               ] node name [5awoLm1] derived from node ID [5awoLm1yQ0yOkBdLy3FTzQ]; set [node.name] to override
[2018-10-12T18:11:34,499][INFO ][o.e.n.Node               ] version[5.4.1], pid[411], build[2cfe0df/2017-05-29T16:05:51.443Z], OS[Linux/3.10.0-862.11.6.el7.x86_64/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_131/25.131-b11]
[2018-10-12T18:11:34,499][INFO ][o.e.n.Node               ] JVM arguments [-Xms22g, -Xmx22g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+DisableExplicitGC, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Djdk.io.permissionsUseCanonicalPath=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -Dio.netty.recycler.maxCapacityPerThread=0, -Des.path.home=/opt/elasticsearch]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [aggs-matrix-stats]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [ingest-common]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [lang-expression]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [lang-groovy]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [lang-mustache]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [lang-painless]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [percolator]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [reindex]
[2018-10-12T18:11:35,239][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [transport-netty3]
[2018-10-12T18:11:35,240][INFO ][o.e.p.PluginsService     ] [5awoLm1] loaded module [transport-netty4]
[2018-10-12T18:11:35,240][INFO ][o.e.p.PluginsService     ] [5awoLm1] no plugins loaded
[2018-10-12T18:11:36,442][INFO ][o.e.d.DiscoveryModule    ] [5awoLm1] using discovery type [zen]
[2018-10-12T18:11:40,530][INFO ][o.e.n.Node               ] initialized
[2018-10-12T18:11:40,530][INFO ][o.e.n.Node               ] [5awoLm1] starting ...
[2018-10-12T18:11:40,647][INFO ][o.e.t.TransportService   ] [5awoLm1] publish_address {172.18.0.2:9300}, bound_addresses {[::]:9300}
[2018-10-12T18:11:40,651][INFO ][o.e.b.BootstrapChecks    ] [5awoLm1] bound or publishing to a non-loopback or non-link-local address, enforcing bootstrap checks
[2018-10-12T18:11:43,691][INFO ][o.e.c.s.ClusterService   ] [5awoLm1] new_master {5awoLm1}{5awoLm1yQ0yOkBdLy3FTzQ}{ycEDt_K0SOG9xEQrhNpN5Q}{172.18.0.2}{172.18.0.2:9300}, reason: zen-disco-elected-as-master ([0] nodes joined)
[2018-10-12T18:11:43,703][INFO ][o.e.h.n.Netty4HttpServerTransport] [5awoLm1] publish_address {172.18.0.2:9200}, bound_addresses {[::]:9200}
[2018-10-12T18:11:43,704][INFO ][o.e.n.Node               ] [5awoLm1] started
[2018-10-12T18:11:46,554][INFO ][o.e.m.j.JvmGcMonitorService] [5awoLm1] [gc][6] overhead, spent [257ms] collecting in the last [1s]
[2018-10-12T18:11:47,555][INFO ][o.e.m.j.JvmGcMonitorService] [5awoLm1] [gc][7] overhead, spent [300ms] collecting in the last [1s]
[2018-10-12T18:11:49,556][INFO ][o.e.m.j.JvmGcMonitorService] [5awoLm1] [gc][9] overhead, spent [290ms] collecting in the last [1s]
[2018-10-12T18:11:50,720][WARN ][o.e.c.s.ClusterService   ] [5awoLm1] failed to notify ClusterStateApplier
org.elasticsearch.ElasticsearchParseException: Need [file], [id], or [inline] parameter to refer to scripts
  at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:233) ~[elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.ingest.common.ScriptProcessor$Factory.create(ScriptProcessor.java:107) ~[?:?]
  at org.elasticsearch.ingest.common.ScriptProcessor$Factory.create(ScriptProcessor.java:87) ~[?:?]
  at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:307) ~[elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.ingest.ConfigurationUtils.readProcessorConfigs(ConfigurationUtils.java:253) ~[elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.ingest.Pipeline$Factory.create(Pipeline.java:122) ~[elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.ingest.PipelineStore.innerUpdatePipelines(PipelineStore.java:81) ~[elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.ingest.PipelineStore.applyClusterState(PipelineStore.java:68) ~[elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.cluster.service.ClusterService.callClusterStateAppliers(ClusterService.java:815) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.cluster.service.ClusterService.publishAndApplyChanges(ClusterService.java:769) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.cluster.service.ClusterService.runTasks(ClusterService.java:586) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.cluster.service.ClusterService$ClusterServiceTaskBatcher.run(ClusterService.java:262) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:247) [elasticsearch-5.4.1.jar:5.4.1]
  at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:210) [elasticsearch-5.4.1.jar:5.4.1]
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
  at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
[2018-10-12T18:11:55,569][INFO ][o.e.g.GatewayService     ] [5awoLm1] recovered [1113] indices into cluster_state