Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Minimal elasticsearch setup for prosody logging
  1. Disable automatic index creation in Elasticsearch

    (except those ES needs internally)

    elasticsearch.yml

    action.auto_create_index: ".watches,.triggered_watches,.watcher-history-*"
  2. Create a lifecycle policy for automatic rollover and deletion

    PUT http://localhost:9200/_ilm/policy/fluent-bit

    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_age": "1d",
                "max_size": "100mb"
              },
              "set_priority": {
                "priority": 100
              }
            },
            "min_age": "0ms"
          },
          "warm": {
            "actions": {
              "set_priority": {
                "priority": 50
              },
              "forcemerge": {
                "max_num_segments": 1
              },
              "migrate": {
                "enabled": false
              },
              "readonly": {}
            }
          },
          "delete": {
            "min_age": "14d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }
  3. Create index template for your indices. Mine are called fluent-bit-* (with * then being a six-digit numeric suffix), so that’s what I’m going to show here:

    PUT http://localhost:9200/_index_template/fluent-bit

    {
      "priority": 100,
      "template": {
        "settings": {
          "number_of_replicas": 0,
          "number_of_shards": 1,
          "index": {
            "lifecycle": {
              "name": "fluent-bit",
              "rollover_alias": "fluent-bit"
            }
          }
        },
        "mappings": {
          "dynamic": false,
          "properties": {
            "@timestamp": {
              "type": "date"
            },
            "message": {
              "type": "text",
              "fields": {
                "match": {
                  "type": "keyword"
                }
              }
            },
            "source": {
              "type": "text",
              "fields": {
                "match": {
                  "type": "keyword"
                }
              }
            },
            "args": {
              "type": "text",
              "fields": {
                "match": {
                  "type": "keyword"
                }
              }
            },
            "level": {
              "type": "keyword"
            }
          }
        }
      },
      "index_patterns": [
        "fluent-bit-*"
      ]
    }
  4. Create the first index for fluentd to push into

    PUT http://localhost:9200/fluent-bit-000001

    {
      "aliases": {
        "fluent-bit": {}
      }
    }
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment