[ELASTICSEARCH 101] Elasticsearch 101 #bash #curl #elasticsearch #101

ELASTICSEARCH-101.md

ELASTICSEARCH 101

• Purge unassigned shards curl -XGET http://localhost:9200/_cat/shards | grep UNASSIGNED | awk {'print $1'} | xargs -i curl -XDELETE "http://localhost:9200/{}"

• Increase numbers of shards https://www.elastic.co/guide/en/elasticsearch/reference/7.17/size-your-shards.html

PUT _cluster/settings
{
  "persistent" : {
    "cluster.routing.allocation.total_shards_per_node" : 100000
  }
}

• Query nested key for value

# query.json
{
  "query": {
    "match": {
      "source.agent.name": "xxxx-xxxx-xxxx"
    }
  }
}
curl -XGET "http://your_elasticsearch_host:9200/_search" -H 'Content-Type: application/json' -d @query.json

• Reindex data to reduces number of shards

POST _reindex
{
  "source": {
    "index": "my-index-2099.10.*"
  },
  "dest": {
    "index": "my-index-2099.10"
  }
}

• Clean reindex

# First of all: enable blocks write to enable clonage
PUT /my_index/_settings
{
  "settings": {
    "index.blocks.write": true
  }
}

# clone index into a temporary index
POST /my_index/_clone/my_index-000001  

# Copy back all documents in the original index to force their reindexetion
POST /_reindex
{
  "source": {
    "index": "my_index-000001"
  },
  "dest": {
    "index": "my_index"
  }
}

# Disable blocks write
PUT /my_index/_settings
{
  "settings": {
    "index.blocks.write": false
  }
}

# Finaly delete the temporary index
DELETE my_index-000001
```# First of all: enable blocks write to enable clonage
PUT /my_index/_settings
{
  "settings": {
    "index.blocks.write": true
  }
}

# clone index into a temporary index
POST /my_index/_clone/my_index-000001  

# Copy back all documents in the original index to force their reindexetion
POST /_reindex
{
  "source": {
    "index": "my_index-000001"
  },
  "dest": {
    "index": "my_index"
  }
}

# Disable blocks write
PUT /my_index/_settings
{
  "settings": {
    "index.blocks.write": false
  }
}

# Finaly delete the temporary index
DELETE my_index-000001

• Get stats of index curl -X GET 'http://127.0.0.1:9200/system-2023-01/_stats' | jq .

• Help avoiding errors with a simple copy-your-config website https://checkups.opster.com/checkup/input

• Run in debug mode (foreground) /usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash" -e

• Post data to Elastic (json format) curl -H "Content-Type: application/json" -XPOST "http://localhost:9200/indexname/typename/optionalUniqueId" -d "{ \"Hello" : \"world\"}"

• Read data to Elastic (json format) curl -H "Content-Type: application/json" -XPOST "http://localhost:9200/_search" -d'{"query": { "match_all": {} }}'

• Find specific data (json format) curl -H "Content-Type: application/json" -XPOST "http://localhost:9200/_search" -d'{"query": {"query_string": {"query": "Hello"}}}'

• List all indices curl -X GET 'http://127.0.0.1:9200/_cat/indices/*?v&s=index'

• Get last 1000 documents from an index curl -X GET 'http://127.0.0.1:9200/myindex/_search?size=1000&pretty=true'

• Get index info (ILM, ect...) curl -X GET 'http://127.0.0.1:9200/myindex'

• Retrieve all mappings on indices curl -XGET http://localhost:9200/_all/_mapping

• Get current usage of Elasticsearch threads curl -XGET 'localhost:9600/_node/hot_threads?human=true'

• Delete specific index curl -X DELETE 'http://127.0.0.1:9200/graylog_deflector'

• Unlock flood-state when disk is nearly full curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }' curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'

• Repair Elasticsearch wrong indexes with MongoDB

mongo --port 27017 --authenticationDatabase "graylog"
use graylog
show collections
db.index_ranges.drop()
exit

• Complex query (for listing last added data in index)

cat << EOF > query.json
{
  "size": 1,
  "sort": {"@timestamp":"desc"},
  "query": {
    "match_all": {}
  }
}
EOF
curl -XPOST http://localhost:9200/index/_search -H 'Content-Type: application/json' -d @query.json

HOT-WARM-COLD.md

HOT-WARM-COLD 101

• Enabling hot, warm or cold on nodes (old method)

bin/elasticsearch -Enode.attr.data=hot
bin/elasticsearch -Enode.attr.data=warm
bin/elasticsearch -Enode.attr.data=cold

• Enabling hot, warm or cold on nodes (v7.10+)

node.name: node-a-hot 
node.roles: ["master", "data_hot", "data_content", "ingest"]

• Push basic ILM strategy

PUT /_ilm/policy/my_policy
{
  "policy":{
    "phases":{
      "hot":{
        "actions":{
          "rollover":{
            "max_size":"50gb",
            "max_age":"30d"
          }
        }
      }
    }
  }
}

• Push template to Elastic

PUT _template/my_template
{
  "index_patterns": ["test-*"], 
  "settings": {
    "index.lifecycle.name": "my_policy", 
    "index.lifecycle.rollover_alias": "test-alias" 
  }
}

• Push hot-warm-cold ILM strategy

PUT _ilm/policy/hot-warm-cold-delete-60days
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size":"50gb",
            "max_age":"30d"
          },
          "set_priority": {
            "priority": 50
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "forcemerge": {
            "max_num_segments": 1
          },
          "shrink": {
            "number_of_shards": 1
          },
          "allocate": {
            "require": {
              "data": "warm"
            }
          },
          "set_priority": {
            "priority": 25
          }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "set_priority": {
            "priority": 0
          },
          "freeze": {},
          "allocate": {
            "require": {
              "data": "cold"
            }
          }
        }
      },
      "delete": {
        "min_age": "60d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

• Another one

PUT _template/hot-warm-cold-delete-60days-template
{
  "order": 10,
  "index_patterns": ["logstash-*", "metricbeat-*", "filebeat-*"],
  "settings": {
    "index.routing.allocation.require.data": "hot",
    "index.lifecycle.name": "hot-warm-cold-delete-60days"
  }
}

• Configure beats to use ILM

output.elasticsearch:
  ilm.enabled: true

• Configure logstash to use ILM

output {
  elasticsearch {  
    ilm_enabled => true
  }
}

• Use template for index automated creation in logstash

output {
    elasticsearch {
     hosts => ["elasticserver:9200"]
     index => "newIndexName-%{+YYYY.MM.dd}"
     template => "pathtotemplate.json"
     template_name => "newIndexName-*"
     template_overwrite => true
    }
    stdout{}
}

• Full ILM policy

PUT /_ilm/policy/my-data-lifecycle 
{
  "policy" : { 
    "phases" : { 
      "hot" : { 
        "actions" : { 
          "rollover" : { 
            "max_size" : "50gb", 
            "max_age" : "3d" 
          } 
        } 
      }, 
      "warm" : { 
        "min_age" : "5d", 
        "actions" : { 
          "shrink" : { 
            "number_of_shards" : 1 
          } 
        } 
      }, 
      "cold" : { 
        "min_age" : "7d", 
        "actions" : { 
          "searchable_snapshot" : { 
            "snapshot_repository" : "my-repository" 
          } 
        } 
      }, 
      "delete" : { 
        "min_age" : "365d", 
        "actions" : { 
          "delete" : { } 
        } 
      } 
    } 
  } 
}

• Push index template

PUT /_index_template/my-lifecycle-template 
{ 
  "index_patterns": ["test-index"], 
  "data_stream" :{}, 
  "template": { 
    "settings": { 
      "index.lifecycle.name": "my-data-lifecycle", 
      "index.number_of_shards": 2 
    } 
  } 
}

• Push test data

POST /test-index/_doc?op_type=create 
{ 
  "message": "test document", 
  "@timestamp": "2020-01-12" 
}