Elastic | Performance tuning for big data import

elastic_bulk-import-settings.sh

##
# Temporary Speed-up configuration for fast import
# references: https://www.elastic.co/blog/performance-considerations-elasticsearch-indexing
# p.s. If you do not want to restart your node/cluster - don't perform 1 and 2 points
##

# 1. Change "ES_HEAP_SIZE" from 50% of memory (default and recommended) to something like 80%
mem=`free -m | grep Mem | awk '{ print $2 }'` && heap=$[ ($mem/1024)*80/100 ]g && \
sudo sed -i "s/ES_HEAP_SIZE=.*/ES_HEAP_SIZE=$heap/" /etc/init.d/elasticsearch

# 2. Change "Index buffer size" to something bigger then default 10%
echo 'indices.memory.index_buffer_size: 50%' | sudo tee >> /etc/elasticsearch/elasticsearch.yml

# 3. Restart elasticsearch (unfortunatelly), because settings in points 1, 2 are not dynamic
sudo service elasticsearch restart

# 4. Disable replicas, refresh, throttle, increase buffer size, disable warmer
echo '
{
    "index.number_of_replicas": "0",
    "index.refresh_interval": "120s",
    "index.warmer.enabled": "false"
}' > off-throttle
curl -s -XGET 'localhost:9200/_cat/indices?v' | egrep -o "logstash-[0-9]{4}\.[0-9]{2}\.[0-9]{2}" | while read index; do \
curl -XPUT "localhost:9200/$index/_settings" -d @off-throttle; done

# 5. Temporary cluster settings (before first restart)
curl -s -XPUT 'localhost:9200/_cluster/settings' -d' { "transient" : { "indices.store.throttle.type": "none" } }'

##
# When inporting is completed - Go back to defaults
##

# 1. Change heap back to 50%
mem=`free -m | grep Mem | awk '{ print $2 }'` && heap=$[ ($mem/1024)*50/100 ]g && \
sudo sed -i "s/ES_HEAP_SIZE=.*/ES_HEAP_SIZE=$heap/" /etc/init.d/elasticsearch

# 2. Remove configuration for "index buffer"
sudo sed -i '/indices.memory.index_buffer_size: 50%/d' /etc/elasticsearch/elasticsearch.yml

# 3. Change back to defaults index settings
echo '
{
    "index.number_of_replicas": "1",
    "index.refresh_interval": "5s",
    "index.warmer.enabled": "true"
 }' > on-throttle
curl -s -XGET 'localhost:9200/_cat/indices?v' | egrep -o "logstash-[0-9]{4}\.[0-9]{2}\.[0-9]{2}" | while read index; do \
curl -XPUT "localhost:9200/$index/_settings" -d @on-throttle; done

# 4. Restart elasticsearch (unfortunatelly), because settings in points 1, 2 are not dynamic
sudo service elasticsearch restart