logstash configuration example for kafka consuming logs

logstash-local.conf

input {
  syslog {
    type => "haproxy-access"
    port => 5000
  }
  log4j {
    mode => "server"
    host => "0.0.0.0"
    port => 3456
    type => "zookeeper"
  }
}

output {
      if [type] == "haproxy-access"  {
        kafka {
          bootstrap_servers => "zookeeperkafka:9092"
          topic_id => "logstash.haproxy"
        }
      }
      else if [type] == "zookeeper" {
        kafka {
          bootstrap_servers => "zookeeperkafka:9092"
          topic_id => "logstash.zookeeper"
        }
      }
}

logstash-server.conf

input {
  kafka {
    type => "haproxy-access"
    zk_connect => "zookeeperkafka:2181"
    topic_id => "logstash.haproxy"
  }
  kafka {
    type => "zookeeper"
    zk_connect => "zookeeperkafka:2181"
    topic_id => "logstash.zookeeper"
  }
}

filter {
  if [type] == "haproxy-access"  {
    grok {
      match => ["message", "%{HAPROXYHTTP}"]
    }
    mutate {
      add_tag => ["HAPROXY"]
    }
  }
  else if [type] == "zookeeper" {
    grok {
      match => { "logger_name" => "org\.apache\.zookeeper\.server\.quorum.*" }
      add_tag => ["LOOOOOOOOOOOL"]
    }
  }
}

output {
  if [type] == "haproxy-access"  {
    elasticsearch {
      hosts => ["elasticsearch:9200"]
      index => "haproxy-%{+YYYY.MM.dd}"
    }
  }
  else if [type] == "zookeeper" {
    elasticsearch {
      hosts => ["elasticsearch:9200"]
      index => "zookeeper-%{+YYYY.MM.dd}"
    }
  }
}