Minimal elasticsearch setup for prosody logging

elasticsearch-guide.md

1. Disable automatic index creation in Elasticsearch

   (except those ES needs internally)

   elasticsearch.yml

   action.auto_create_index: ".watches,.triggered_watches,.watcher-history-*"

2. Create a lifecycle policy for automatic rollover and deletion

   PUT http://localhost:9200/_ilm/policy/fluent-bit

   {
     "policy": {
       "phases": {
         "hot": {
           "actions": {
             "rollover": {
               "max_age": "1d",
               "max_size": "100mb"
             },
             "set_priority": {
               "priority": 100
             }
           },
           "min_age": "0ms"
         },
         "warm": {
           "actions": {
             "set_priority": {
               "priority": 50
             },
             "forcemerge": {
               "max_num_segments": 1
             },
             "migrate": {
               "enabled": false
             },
             "readonly": {}
           }
         },
         "delete": {
           "min_age": "14d",
           "actions": {
             "delete": {}
           }
         }
       }
     }
   }

3. Create index template for your indices. Mine are called fluent-bit-* (with * then being a six-digit numeric suffix), so that’s what I’m going to show here:

   PUT http://localhost:9200/_index_template/fluent-bit

   {
     "priority": 100,
     "template": {
       "settings": {
         "number_of_replicas": 0,
         "number_of_shards": 1,
         "index": {
           "lifecycle": {
             "name": "fluent-bit",
             "rollover_alias": "fluent-bit"
           }
         }
       },
       "mappings": {
         "dynamic": false,
         "properties": {
           "@timestamp": {
             "type": "date"
           },
           "message": {
             "type": "text",
             "fields": {
               "match": {
                 "type": "keyword"
               }
             }
           },
           "source": {
             "type": "text",
             "fields": {
               "match": {
                 "type": "keyword"
               }
             }
           },
           "args": {
             "type": "text",
             "fields": {
               "match": {
                 "type": "keyword"
               }
             }
           },
           "level": {
             "type": "keyword"
           }
         }
       }
     },
     "index_patterns": [
       "fluent-bit-*"
     ]
   }

4. Create the first index for fluentd to push into

   PUT http://localhost:9200/fluent-bit-000001

   {
     "aliases": {
       "fluent-bit": {}
     }
   }