Filebeat Squid Proxy Access Log Parsing

access.log

1348870236.160      0 192.168.0.35 TCP_DENIED/403 3293 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870236.273      0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870236.386      0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870236.499      0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870237.550      0 192.168.0.35 TCP_DENIED/403 3269 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.4.6/misc/AcrobatUpd946_all_incr.msp - NONE/- text/html
1348870274.248  59875 192.168.0.35 TCP_MISS/503 0 CONNECT client84.dropbox.com:443 - DIRECT/- -
1348870284.249  59872 192.168.0.35 TCP_MISS/503 0 CONNECT client62.dropbox.com:443 - DIRECT/- -
1348870285.249  59406 192.168.0.35 TCP_MISS/503 0 CONNECT d.dropbox.com:443 - DIRECT/- -
1348870294.249  59868 192.168.0.35 TCP_MISS/503 0 CONNECT client20.dropbox.com:443 - DIRECT/- -

filebeat.squid.yml

filebeat.inputs:
- paths: /Users/akroh/Downloads/squid-access.log
  pipeline: filebeat-squid

processors:
- dissect:
    tokenizer: '%{timestamp->} %{event.duration} %{source.ip} %{squid.access.result_code}/%{http.response.status_code} %{http.response.bytes} %{http.request.method} %{url.original} %{user.name} %{squid.access.peer.status}/%{squid.access.peer.host} %{http.response.content_type}'
    target_prefix: ''
- timestamp:
    field: timestamp
    layouts:
      - UNIX
- convert:
    fields:
      - {from: squid.access.peer.host, to: destination.ip, type: ip}
    fail_on_error: false
    ignore_missing: true
- dissect:
    field: url.original
    tokenizer: '%{url.scheme}://%{url.domain}/%{url.path}'
    target_prefix: ''
    when.regexp.url.original: '^\w+://'
- dissect:
    field: url.original
    tokenizer: '%{url.domain}:%{url.port}'
    target_prefix: ''
    when.regexp.url.original: '.*:\d+'
- registered_domain:
    field: url.domain
    target_field: url.registered_domain
    ignore_missing: true
    ignore_failure: true

output.elasticsearch:
  hosts: ['http://localhost:9200/']

gistfile1.txt

PUT _ingest/pipeline/filebeat-squid
{
  "description": "Pipeline for Filebeat Squid Access",
  "processors": [
    {
      "geoip": {
        "if": "ctx.source?.geo == null",
        "field": "source.ip",
        "target_field": "source.geo",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "if": "ctx.destination?.geo == null",
        "field": "destination.ip",
        "target_field": "destination.geo",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "source.ip",
        "target_field": "source.as",
        "properties": [
          "asn",
          "organization_name"
        ],
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "destination.ip",
        "target_field": "destination.as",
        "properties": [
          "asn",
          "organization_name"
        ],
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "source.as.asn",
        "target_field": "source.as.number",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "source.as.organization_name",
        "target_field": "source.as.organization.name",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "destination.as.asn",
        "target_field": "destination.as.number",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "destination.as.organization_name",
        "target_field": "destination.as.organization.name",
        "ignore_missing": true
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error.message",
        "value": "{{ _ingest.on_failure_message }}"
      }
    }
  ]
}