cluster settings

{
   "persistent": {},
   "transient": {}
}

elasticsearch.yml

# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
#       Before you set out to tweak and tune the configuration, make sure you
#       understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: elasticsearch_vangp1
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
#node.name: node-1
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /usr/local/var/lib/elasticsearch/
#
# Path to log files:
#
path.logs: /usr/local/var/log/elasticsearch/
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
network.host: 0.0.0.0
#
# Set a custom port for HTTP:
#
#http.port: 9200
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
#
# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
#
#discovery.zen.minimum_master_nodes:
#
# For more information, consult the zen discovery module documentation.
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, consult the gateway module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

gistfile1.txt

[2019-05-30T16:47:30,236][WARN ][o.e.i.c.IndicesClusterStateService] [pSuUtbN] [[test_20190224060157_43mk275ixpqyd2gc_data_backlog_viz_ns_default][0]] marking and sending shard failed due to [failed recovery]
org.elasticsearch.indices.recovery.RecoveryFailedException: [test_20190224060157_43mk275ixpqyd2gc_data_backlog_viz_ns_default][0]: Recovery failed on {pSuUtbN}{pSuUtbN-T4e0Ovf03sUS6g}{Zd6-BsorS7ukxeouzioKjQ}{127.0.0.1}{127.0.0.1:9300}
	at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2394) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-6.8.0.jar:6.8.0]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_192]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_192]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_192]
Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway
	at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:445) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:310) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1737) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2390) ~[elasticsearch-6.8.0.jar:6.8.0]
	... 4 more
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine
	at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:208) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1494) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1444) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:440) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:310) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1737) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2390) ~[elasticsearch-6.8.0.jar:6.8.0]
	... 4 more
Caused by: java.nio.file.FileSystemException: /usr/local/var/lib/elasticsearch/nodes/0/indices/-qUAXIl3S5SM0PZN9wGKQw/0/translog/translog-42.ckp: Too many open files
	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) ~[?:?]
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
	at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_192]
	at java.nio.file.Files.newByteChannel(Files.java:407) ~[?:1.8.0_192]
	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77) ~[lucene-core-7.7.0.jar:7.7.0 8c831daf4eb41153c25ddb152501ab5bae3ea3d5 - jimczi - 2019-02-04 23:16:28]
	at org.elasticsearch.index.translog.Checkpoint.read(Checkpoint.java:188) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:243) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.translog.Translog.<init>(Translog.java:189) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:481) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:196) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1494) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1444) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:440) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:310) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1737) ~[elasticsearch-6.8.0.jar:6.8.0]
	at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2390) ~[elasticsearch-6.8.0.jar:6.8.0]