logstash-biz-logs.conf

logstash-biz-logs.conf

input {
    beats {
        port => 5044
    }
}

filter {
    if [log_type] == "php_slow" {
        # php 慢查询日志
        ruby {
            code => "lines = event.get('message').split('
')

            occur_time, pool_name, pid = lines[0].match(/\[(.*?)\]  \[pool (.*?)\] pid (\d+)/).captures
            script_filename, = lines[1].match(/script_filename = (.*?)$/).captures

            stacks = lines[2..-1].map {|line| line[21..-1]}

            event.set('stacks', stacks)
            event.set('script_filename', script_filename)
            event.set('occur_time', occur_time)
            event.set('pool_name', pool_name)
            event.set('pid', pid)
            "
        }
        date {
            match => ["occur_time", "dd-MMM-yyyy HH:mm:ss"]
            target => "occur_time"
        }
    } else if [log_type] == "mysql_slow" {
        # mysql 慢查询日志
        grok {
            match => {
                "message" => "(?m)^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IPV4:clientip})?\]\s+Id:\s+%{NUMBER:row_id:int}\n#\s+Query_time:\s+%{NUMBER:query_time:float}\s+Lock_time:\s+%{NUMBER:lock_time:float}\s+Rows_sent:\s+%{NUMBER:rows_sent:int}\s+Rows_examined:\s+%{NUMBER:rows_examined:int}\n\s*(?:use %{DATA:database};\s*\n)?SET\s+timestamp=%{NUMBER:occur_time};\n\s*(?<sql>(?<action>\w+)\b.*;)\s*(?:\n#\s+Time)?.*$"
            }
            remove_field => ["message"]
        }
        date {
            match => ["occur_time", "UNIX", "YYYY-MM-dd HH:mm:ss"]
            target => "occur_time"
        }
    } else if [log_type] == "nginx_error" {
        # nginx 错误日志
        grok {
            match => {
                "message" => "(?<occur_time>%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}) \[%{LOGLEVEL:log_level}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:error_msg}"
            }
            remove_field => ["message"]
        }
        date {
            match => ["occur_time", "yyyy-MM-dd HH:mm:ss"]
        }
    }
}

output {
    if [log_type] == "php_slow" {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "php-slow-%{+YYYY.MM.dd}"
        }
    } else if [log_type] == "mysql_slow" {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "mysql-slow-%{+YYYY.MM.dd}"
        }
    } else if [log_type] == "nginx_error" {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "nginx-error-%{+YYYY.MM.dd}"
        }
    }

    #stdout { codec => rubydebug }
}