Bosun Data in ES PoC

docker.sh

docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:5.5.3 && docker run -d -p 5601:5601 --net=host -e "ELASTICSEARCH_URL=http://localhost:9200" docker.elastic.co/kibana/kibana:5.5.3

hackyLoad.sh

cat incidents.json | jq -c '.[] | {"index": {"_index": "incidents", "_type": "incident", "_id": .Id}}, .' | curl -u elastic:changeme -XPOST localhost:9200/_bulk --data-binary @-

idump.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"strings"
	"time"

	"bosun.org/cmd/bosun/database"
	"bosun.org/models"
	"bosun.org/opentsdb"
	"github.com/garyburd/redigo/redis"
)

var (
	flagRedis   = flag.String("r", "", "redis host and port, ie localhost:1234")
	flagOutFile = flag.String("f", "incidents.json", "file to write array of incidents to, default is incidents.json")
)

func main() {
	flag.Parse()
	da := database.NewDataAccess(*flagRedis, true, 0, "")
	c := da.Get()
	defer c.Close()

	ids, err := int64s(c.Do("LRANGE", "allIncidents", 0, -1))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(len(ids))
	incidents := []*models.IncidentState{}
	missingCount := 0
	for _, id := range ids {
		incident, err := da.State().GetIncidentState(id)
		if err != nil {
			missingCount++
			continue
		}
		incidents = append(incidents, incident)
	}
	fmt.Println(missingCount)
	eIncidents := []ESIncident{}
	for _, i := range incidents {
		ei := ESIncident{
			Id:            i.Id,
			AlertKey:      string(i.AlertKey),
			Name:          i.Alert,
			Start:         i.Start,
			End:           i.End,
			Subject:       i.Subject,
			Notifications: i.Notifications,
			EventCount:    len(i.Events),
			TagSet:        i.Group(),
		}
		for _, e := range i.Events {
			ei.EventStatusList = append(ei.EventStatusList, e.Status)
		}
		users := make(map[string]bool)
		for _, a := range i.Actions {
			ei.ActionMessages = append(ei.ActionMessages, a.Message)
			switch a.Type {
			case models.ActionAcknowledge:
				ei.AckUser = a.User
				ei.SecondsToAck = int64(a.Time.Sub(ei.Start).Seconds())
			case models.ActionClose:
				ei.CloseUser = a.User
				ei.SecondsToClose = int64(a.Time.Sub(ei.Start).Seconds())
			}
			users[a.User] = true
		}
		for user := range users {
			ei.Users = append(ei.Users, user)
		}
		eIncidents = append(eIncidents, ei)
	}
	j, err := json.Marshal(eIncidents)
	if err != nil {
		log.Fatal(err)
	}
	err = ioutil.WriteFile(*flagOutFile, j, 0600)
	if err != nil {
		log.Fatal(err)
	}
}

type ESIncident struct {
	Id              int64
	Start           time.Time
	End             *time.Time
	Name            string
	AlertKey        string
	Subject         string
	Notifications   []string
	EventStatusList []models.Status
	EventCount      int
	ActionMessages  []string
	Users           []string
	SecondsToAck    int64
	AckUser         string
	SecondsToClose  int64
	CloseUser       string
	opentsdb.TagSet
}

func int64s(reply interface{}, err error) ([]int64, error) {
	if err != nil {
		return nil, err
	}
	ints := []int64{}
	values := []string{}
	rawValues, err := redis.Values(reply, err)
	if err != nil {
		return ints, err
	}
	if err := redis.ScanSlice(rawValues, &values); err != nil {
		return ints, err
	}
	for _, v := range values {
		sp := strings.Split(v, ":")
		if len(sp) != 3 {
			return ints, fmt.Errorf("bad incident value in allIncidents: %v", v)
		}
		i, err := strconv.ParseInt(sp[0], 0, 64)
		if err != nil {
			return ints, fmt.Errorf("bad id value for incident in allIncidents: %v", sp[0])
		}
		ints = append(ints, i)
		//if len(ints) > 100 {
		//	break
		//}
	}
	return ints, nil
}

/*
CREATE TABLE i (
    id UUID DEFAULT uuid_v4()::UUID PRIMARY KEY,
    incidents JSONB
  );
*/

makeIndex.sh

curl -u elastic:changeme -X PUT "localhost:9200/incidents" -H 'Content-Type: application/json' --data-binary @mapping.json

mapping.json

{
    "settings": {
        "number_of_shards": 1,
        "number_of_replicas": 0
    },
    "mappings": {
        "incident": {
            "properties": {
                "Start": {
                    "type": "date"
                },
                "End": {
                    "type": "date"
                },
                "Name": {
                    "type": "keyword"
                },
                "AlertKey": {
                    "type": "keyword"
                },
                "Subject": {
                    "type": "text"
                },
                "Notifications": {
                    "type": "keyword"
                },
                "EventStatusList": {
                    "type": "keyword"
                },
                "EventCount": {
                    "type": "integer"
                },
                "ActionMessages": {
                    "type": "text"
                },
                "Users": {
                    "type": "keyword"
                },
                "SecondsToAck": {
                    "type": "integer"
                },
                "AckUser": {
                    "type": "keyword"
                },
                "SecondsToClose": {
                    "type": "integer"
                },
                "CloseUser": {
                    "type": "keyword"
                }
            }
        }
    }
}