Logstash and Logstash-Forwarder config for GitLab logs

gitlab2elk.yml

# YAML config for these Ansible roles:
#    https://github.com/LaneCommunityCollege/aspects_logstash
#    https://github.com/LaneCommunityCollege/aspects_logstash_forwarder
# 
# Since it's just straight config blocks, you should be able to just copy and paste what you need if you don't use
# those Ansible roles.
#
# End result is multiline logs combined into one, and dates are parsed correctly.

aspects_logstash_rules:
  patterns:
    gitlabpatterns: |
      GLAPPLOGTIME (?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]),\s\d{4}\s(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])
      GLABTIME %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}
  filters:
    gitlabapplicationlog: |
      if [type] == "gitlabapplicationlog"{
        grok { 
          match => ["message", "%{GLAPPLOGTIME:timestamp}"]
          patterns_dir => ["/etc/logstash/patterns"] 
        }
        date {
          match => [ "timestamp", "MMM dd',' yyyy HH:mm" ]
          target => "@timestamp"
        }
      }
    gitlabhostlog: |
      if [type] == "gitlabhostlog"{
        multiline {
          pattern => "^%{GLAPPLOGTIME} "
          negate => true
          what => previous
          patterns_dir => ["/etc/logstash/patterns"] 
        }
        grok { 
          match => ["message", "%{GLAPPLOGTIME:timestamp}"]
          patterns_dir => ["/etc/logstash/patterns"] 
        }
        date {
            match => [ "timestamp", "MMM dd',' yyyy HH:mm" ]
            target => "@timestamp"
          }
      }
    gitlabproductionlog: |
      if [type] == "gitlabproductionlog"{
        multiline {
          pattern => "Started"
          negate => true
          what => previous
        }
        grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"]
          patterns_dir => ["/etc/logstash/patterns"]  }
        date {
            match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ]
            target => "@timestamp"
          }
      }
    gitlabsatlog: |
      if [type] == "gitlabsatlog"{
        grok { 
          match => ["message", "%{GLAPPLOGTIME:timestamp}"]
          patterns_dir => ["/etc/logstash/patterns"] 
        }
        date {
            match => [ "timestamp", "MMM dd',' yyyy HH:mm" ]
            target => "@timestamp"
          }
      }
    gitlabsidekiqlog: |
      if [type] == "gitlabsidekiqlog"{
        multiline {
          pattern => "^%{TIMESTAMP_ISO8601}"
          negate => true
          what => previous
        }
        grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"] }
        date {
            match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ss.SSSZ" ]
            target => "@timestamp"
          }
      }
    gitlabunicornerrlog: |
      if [type] == "gitlabunicornerrlog"{
        grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"] }
        date {
            match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ss.SSSSSS" ]
            target => "@timestamp"
          }
      }
    gitlabunicornoutlog: |
      if [type] == "gitlabunicornoutlog"{
        grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"] }
        date {
            match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ssZ" ]
            target => "@timestamp"
          }
      }

aspects_logstash_forwarder_inputs:
  gitlabproductionlog: |
    { 
        "paths": ["/home/git/gitlab/log/production.log"], "fields": { "type": "gitlabproductionlog"}
    }
  gitlabunicornerrlog: |
    { 
        "paths": ["/home/git/gitlab/log/unicorn.stderr.log"], "fields": { "type": "gitlabunicornerrlog"}
    }
  gitlabunicornoutlog: |
    { 
        "paths": ["/home/git/gitlab/log/unicorn.stdout.log"], "fields": { "type": "gitlabunicornoutlog"}
    }
  gitlabhostlog: |
    { 
        "paths": ["/home/git/gitlab/log/githost.log"], "fields": { "type": "gitlabhostlog"}
    }
  gitlabsidekiqlog: |
    { 
        "paths": ["/home/git/gitlab/log/sidekiq.log"], "fields": { "type": "gitlabsidekiqlog"}
    }
  gitlabsatlog: |
    { 
        "paths": ["/home/git/gitlab/log/satellites.log"], "fields": { "type": "gitlabsatlog"}
    }
  gitlabapplicationlog: |
    { 
        "paths": ["/home/git/gitlab/log/application.log"], "fields": { "type": "gitlabapplicationlog"}
    }