Elasticsearch sensible curator

curate.sh

#!/bin/bash

[ -f "$1" ] && . "$1"

[ "$2" ] && ACTION="$2"
[ "$2" ] && OPTIM=""

for ref in $(seq $MAX -1 $MIN)
do
  IDX="${PREFIX}$(date -d @$(($(date +%s)-(${INTERVAL}*${ref}))) +${PATTERN})"
  if [ "$ACTION" ]
  then
    echo -n "Executing ${ACTION} on ${IDX}"
    if [ "$ACTION" = "DELETE" ]
    then
      echo
      curl -s -X${ACTION} "localhost:9200/${IDX}?pretty"
    elif [ "$ACTION" = "SHOW" ]
    then
      echo
      curl -s -XGET "localhost:9200/_cat/indices/${IDX}?v"
    else
      if [ -z "$JSON" ]
      then
        echo
        curl -s -XPOST "localhost:9200/${IDX}/${ACTION}?pretty"
      else
        echo " using ${JSON}"
        curl -s -XPOST "localhost:9200/${IDX}/${ACTION}?pretty" -H 'Content-Type: application/json' -d @"$(dirname $0)/${JSON}"
      fi
    fi
  fi
  if [ "$OPTIM" = "y" ]
  then
    echo Optimizing ${IDX}
    curl -s -XPOST "localhost:9200/${IDX}/_forcemerge?max_num_segments=1&pretty"
  fi
done

delete-old.sh

INTERVAL=86400
MIN=60
MAX=70
PREFIX=logstash-appdata-
PATTERN=%Y.%m.%d
ACTION=_delete_by_query
JSON=old_data.json
OPTIM=y

old_data.json

{
  "query": {
    "bool": {
      "minimum_should_match": 1,
      "should": [
        {
          "term": {
            "level.raw": "DEBUG"
          }
        },
        {
          "match": {
            "logger_name": {
              "query": "org.apache.solr.update.processor.LogUpdateProcessor",
              "type": "phrase"
            }
          }
        },
        {
          "match": {
            "ndc": {
              "query": "StockUpdate",
              "type": "phrase"
            }
          }
        },
        {
          "match": {
            "ndc": {
              "query": "BackgroundJobs",
              "type": "phrase"
            }
          }
        }
      ],
      "must_not": {
        "match": {
          "logger_name": {
            "query": "org.jetty.core.filter.LoggingFilter",
            "type": "phrase"
          }
        }
      }
    }
  }
}

puppet_elk_curator.pp

class elk::curator($cleanups = {}) {
  create_resources(cron, $cleanups)
  file { '/usr/local/share/curate':
    ensure => present,
    source => "puppet:///modules/${module_name}/elk/curate",
    recurse => true,
  }
}

sample.yaml

elk::curator::cleanups:
  logstash-delete-teststuff:
    command: '/bin/bash /usr/local/share/curate/curate.sh /usr/local/share/curate/teststuff-delete.sh | /usr/bin/logger'
    user:    root
    hour:    5
    minute:  12
  logstash-delete-old:
    command: '/bin/bash /usr/local/share/curate/curate.sh /usr/local/share/curate/delete-old.sh | /usr/bin/logger'
    user:     root
    hour:     4
    minute:   14

teststuff-delete.sh

INTERVAL=86400
MIN=30
MAX=60
PREFIX=logstash-teststuff-
PATTERN=%Y.%m.%d
ACTION=DELETE