Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Filebeat Squid Proxy Access Log Parsing
1348870236.160 0 192.168.0.35 TCP_DENIED/403 3293 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870236.273 0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870236.386 0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870236.499 0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
1348870237.550 0 192.168.0.35 TCP_DENIED/403 3269 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.4.6/misc/AcrobatUpd946_all_incr.msp - NONE/- text/html
1348870274.248 59875 192.168.0.35 TCP_MISS/503 0 CONNECT client84.dropbox.com:443 - DIRECT/- -
1348870284.249 59872 192.168.0.35 TCP_MISS/503 0 CONNECT client62.dropbox.com:443 - DIRECT/- -
1348870285.249 59406 192.168.0.35 TCP_MISS/503 0 CONNECT d.dropbox.com:443 - DIRECT/- -
1348870294.249 59868 192.168.0.35 TCP_MISS/503 0 CONNECT client20.dropbox.com:443 - DIRECT/- -
filebeat.inputs:
- paths: /Users/akroh/Downloads/squid-access.log
pipeline: filebeat-squid
processors:
- dissect:
tokenizer: '%{timestamp->} %{event.duration} %{source.ip} %{squid.access.result_code}/%{http.response.status_code} %{http.response.bytes} %{http.request.method} %{url.original} %{user.name} %{squid.access.peer.status}/%{squid.access.peer.host} %{http.response.content_type}'
target_prefix: ''
- timestamp:
field: timestamp
layouts:
- UNIX
- convert:
fields:
- {from: squid.access.peer.host, to: destination.ip, type: ip}
fail_on_error: false
ignore_missing: true
- dissect:
field: url.original
tokenizer: '%{url.scheme}://%{url.domain}/%{url.path}'
target_prefix: ''
when.regexp.url.original: '^\w+://'
- dissect:
field: url.original
tokenizer: '%{url.domain}:%{url.port}'
target_prefix: ''
when.regexp.url.original: '.*:\d+'
- registered_domain:
field: url.domain
target_field: url.registered_domain
ignore_missing: true
ignore_failure: true
output.elasticsearch:
hosts: ['http://localhost:9200/']
PUT _ingest/pipeline/filebeat-squid
{
"description": "Pipeline for Filebeat Squid Access",
"processors": [
{
"geoip": {
"if": "ctx.source?.geo == null",
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
},
{
"geoip": {
"if": "ctx.destination?.geo == null",
"field": "destination.ip",
"target_field": "destination.geo",
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "destination.ip",
"target_field": "destination.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
}
},
{
"rename": {
"field": "destination.as.asn",
"target_field": "destination.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "destination.as.organization_name",
"target_field": "destination.as.organization.name",
"ignore_missing": true
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment