Needs to be cleaned up and probably doing stupid shit

README.md

Basic port of my old rabbitmq+ruby global log tailer to redis+golang.

Still a go newb but it works. Probably lots of edge cases. Managed to port the bits to publish to a websocket instead of stdout. Plan on cleaning all that up and publishing it as two bits - the service and the client.

This is just a basic client. Totally insecure.

Note you would need to customize this. We have our own keys in our logstash events that are pointless to you.

This tool is primarily for our developers to be able to work with our production logs. They have access to our Kibana install but sometimes being able to just hit the command-line and use natural tools makes more sense. There's also an option to display stacktraces as well which is handy to be able to turn on or off.

All of our applications are poke-able over JMX to crank logging up or down so this combined with that is pretty awesome.

estail.go

package main

import (
	"flag"
	"fmt"
	"encoding/json"
	"github.com/garyburd/redigo/redis"
)

// redis key for psub
// "stream.%{es_environment}.%{es_component}.%{source_host}.%{severity}"

type JavaException struct {
	ExceptionClass string `json:"exception_class"`
	ExceptionMessage string `json:"exception_message"`
	StackTrace string `json:"stack_trace"`
}

// Essentially all the possible keys we have
type LogstashEvent struct {
	Timestamp string `json:"@timestamp"`
	Version int `json:"@version"`
	Environment string `json:"es_environment"`
	Severity string `json:"severity"`
	Input string `json:"ls_input"`
	SourceHost string `json:"source_host"`
	Message string `json:"message"`

	LoggerName string `json:"logger_name,omitempty"` //java apps
	LineNumber string `json:"line_number,omitempty"` // java apps
	ClassName string `json:"class,omitempty"` //java apps
	ThreadName string `json:"thread_name,omitempty"` //java apps
	File string `json:"file,omitempty"` //java apps?
	Level string `json:"level,omitempty"` //java apps
	Method string `json:"method,omitempty"` //java apps
	Component string `json:"es_component"`
	//Mdc map[string]interface{} `json:"mdc,omitempty"` //java apps
	//Ndc []interface{} `json:"ndc,omitempty"` //java apps
	// My shortsightedness has us overloading this json key - sigh
	Exception interface{} `json:"exception"` //java apps

	Program string `json:"program,omitempty"` //syslog
	ProcessId string `json:"processid,omitempty"` //syslog
	Host string `json:"host,omitempty"` //syslog

	UpdatedResources []string `json:"updated_resources,omitempty"` //chef
	ElapsedTime float32 `json:"elapsed_time,omitempty"` //chef
	Success bool `json:"success,omitempty"` //chef
	StartTime string `json:"start_time,omitempty"` //chef
	EndTime string `json:"end_time,omitempty"` //chef
	BackTrace string `json:"backtrack,omitempty"` //chef

}

func main() {
	redisHost := flag.String("host", "127.0.0.1:6379", "the redis host to connect to")
	channel := flag.String("channel", "stream.*", "the pattern to subscribe to")
	with_exceptions := flag.Bool("with-exceptions", false, "Should stacktraces be shown?")
	flag.Parse()

	client,_ := redis.Dial("tcp", *redisHost)
	defer client.Close()
	_, err := client.Do("PING")
	if err != nil {
		fmt.Printf("%s", err)
	}else{
		pubsub := redis.PubSubConn{Conn: client}
		defer pubsub.Close()
		//var event map[string]interface{}
		var c = make(chan string)
		pubsub.PSubscribe(*channel)
		go func() {
			for {
				switch v := pubsub.Receive().(type) {
				case redis.PMessage:
					c <- string(v.Data)
				case redis.Subscription:
					fmt.Printf("Subscription: %s %s %d\n", v.Kind, v.Channel, v.Count)
					if v.Count == 0 {
						return
					}
				case error:
					return
				}
			}


		}()

		for {
			var j LogstashEvent
			val := <-c
			fmt.Sprintf("%v\n", val)
			if err := json.Unmarshal([]byte(val), &j);err != nil {
				fmt.Printf("Unable to unmarshal event")
			} else {
				output := fmt.Sprintf("[%s] %s\t%s", j.Timestamp, j.SourceHost, j.Message)
				if *with_exceptions == true {
                                        // Should probably switch on the type here
					_, chef_ok := j.Exception.(string)
					if chef_ok {
						output += fmt.Sprintf("\n%s\n%s", j.Exception, j.BackTrace)
					}
					java_exception, java_ok := j.Exception.(map[string]interface{})
					if java_ok {
						output += fmt.Sprintf("\n%s", java_exception["stacktrace"])
					}
				}
				fmt.Println(output)
			}
		}
	}
}

logstash-output.conf

output {
        #stdout { debug => true }
	redis { codec => "json" host => "localhost" data_type => "channel" key => "stream.%{es_environment}.%{es_component}.%{source_host}.%{severity}" }
}