Logstash to Piwik

gistfile1.txt

input { stdin { } }

filter {
  grok {
    match => { "message" => "%{COMMONAPACHELOG} %{QS:agent} %{QS:referrer}" }
  }
  
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}

filter {
  if "_grokparsefailure" in [tags] {
    drop { }
  }

  if [request] =~ /(gif|jpg|jpeg|png|bmp|ico|svg|ttf|eot|woff|class|swf|css|js|xml|robots.txt)/ {
    drop { }
  }
}

filter {
  if [response] =~ /50./ {
    drop { }
  }
  if [response] == "404" {
    drop { }
  }
}

filter {
  if [response] == "403" {
    drop { }
  }
}

filter {
  if [response] == "302" {
    drop { }
  }
}

filter {
  mutate {
    gsub => [
      "agent", '\"', ""
    ]
  }
  
  mutate {
    gsub => [
      "referrer", '\"', ""
    ]
  }
}

filter {
  if [agent] =~ /Bot/ {
    drop { }
  }

  if [agent] =~ /bot/ {
    drop { }
  }

  if [agent] =~ /spider/ {
    drop { }
  }

  if [agent] =~ /crawler/ {
    drop { }
  }

  if [agent] =~ /UptimeRobot/ {
    drop { }
  }

  if [agent] =~ /YandexBot/ {
    drop { }
  }

  if [agent] =~ /Googlebot/ {
    drop { }
  }

  if [agent] =~ /yahoo/ {
    drop { }
  }

  if [agent] =~ /check/ {
    drop { }
  }

  if [agent] =~ /Crawler/ {
    drop { }
  }

  if [agent] =~ /dummy connection/ {
    drop { }
  }  
}


filter {
  geoip {
    database => ".//GeoLiteCity.dat"
    source => "clientip"
  }
}

filter {
  environment {
    add_field_from_env => { "env_host" => "env_host" }
  }
  
  environment {
    add_field_from_env => { "idsite" => "idsite" }
  }

}

filter {
  mutate {
    add_field => { 
      "rec"  => "1"
      "apiv" => "1"
      "url"  => "http://%{env_host}%{request}"
      "urlref" => "%{referrer}"
      "cip" => "%{clientip}"
      "cdt" => "%{@timestamp}"
      "ua" => "%{agent}"
      "dp" => "1"
    }
  }
  
  if [geoip] {
    mutate {
      add_field => {
      "country" => "%{[geoip][country_code2]}"
      "region" => "%{[geoip][real_region_name]}"
      "city" => "%{[geoip][city_name]}"
      "lat" => "%{[geoip][latitude]}"
      "long" => "%{[geoip][longitude]}"
      }
    }
  }

  mutate {
    add_field => {
      "_cvar" => '{"1": ["HTTP-code", "%{response}"] }'
    }
  } 
  
  if [url] =~ /\.(7z|aac|arc|arj|asf|asx|avi|bin|csv|deb|dmg|doc|exe|flv|gz|gzip|hqx|jar|mpg|mp2|mp3|mp4|mpeg|mov|movie|msi|msp|odb|odf|odg|odp|ods|odt|ogg|ogv|pdf|phps|ppt|qt|qtm|ra|ram|rar|rpm|sea|sit|tar|tbz|bz2|tbz|tgz|torrent|txt|wav|wma|wmv|wpd|xls|xml|xsd|z|zip|azw3|epub|mobi)$/ {
    mutate {
      add_field => {
        "download" => "%{url}"
      }
    }
  }
  
  mutate {
    gsub => [
      "cdt", " UTC", ""
    ]
  }
  
  }
  
  filter {
  
  mutate {
    remove_field => [
      "@version",
      "@timestamp",
      "message",
      "host",
      "clientip",
      "ident",
      "auth",
      "timestamp",
      "verb",
      "request",
      "httpversion",
      "response",
      "bytes",
      "referrer",
      "agent"
    ]
  }
  
  mutate {
    remove_field => [ 
      "geoip"
    ]
  }
  
  mutate {
    remove_field => ["env_host"]
  }
}

output {
  stdout { codec => json_lines }
}