Skip to content

Instantly share code, notes, and snippets.

@sathieu

sathieu/mod_security.lua

Last active January 7, 2016 12:43
Show Gist options
  • Save sathieu/0d74beef1d7bd393ee09 to your computer and use it in GitHub Desktop.
Save sathieu/0d74beef1d7bd393ee09 to your computer and use it in GitHub Desktop.
Download ZIP
Parse mod_security logs
Raw
mod_security.lua
-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
--[[
Parses the Apache mod_security logs.
See https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats#Audit_Log
Config:
- type (string, optional, defaults to "apache.security"):
Sets the message 'Type' header to the specified value
- user_agent_transform (bool, optional, default false)
Transform the http_user_agent into user_agent_browser, user_agent_version,
user_agent_os.
- user_agent_keep (bool, optional, default false)
Always preserve the http_user_agent value if transform is enabled.
- user_agent_conditional (bool, optional, default false)
Only preserve the http_user_agent value if transform is enabled and fails.
- payload_keep (bool, optional, default false)
Always preserve the original log line in the message payload.
*Example Heka Configuration*
.. code-block:: ini
[ModsecInput]
type = "LogstreamerInput"
log_directory = "/var/log/apache2"
file_match = 'sec\.log'
splitter = "NullSplitter"
decoder = "ModsecDecoder"
[ModsecSplitter]
type = "RegexSplitter"
delimiter = '(-Z--\n\n)'
[ModsecDecoder]
type = "SandboxDecoder"
filename = "lua_decoders/mod_security.lua"
[ModsecDecoder.config]
user_agent_transform = true
*Example Heka Message*
:Timestamp: 2015-11-20 09:30:56 +0000 UTC
:Type: apache.security
:Hostname: test.example.com
:Pid: 0
:Uuid: 6a7686f5-2f1e-4b05-ac87-86687e2745f0
:Logger: ModsecInput
:Payload:
:EnvVersion:
:Severity: 7
:Fields:
| name:"boundary" type:string value:"f9b8c61b"
| name:"transaction_id" type:string value:"Vk9oUAoCMAEAAF5mzKMAAADS"
| name:"server_addr" type:string value:"10.20.30.40" representation:"ipv4"
| name:"server_port" type:double value:443
| name:"remote_addr" type:string value:"1.2.3.4" representation:"ipv4"
| name:"remote_port" type:double value:60009
| name:"request" type:string value:"GET /favicon.ico HTTP/1.1"
| name:"http_accept_encoding" type:string value:"gzip, deflate"
| name:"http_accept_language" type:string value:"fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3"
| name:"http_accept" type:string value:"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
| name:"http_connection" type:string value:"keep-alive"
| name:"http_cookie" type:string value:"_ga=GA1.2.1920814331.1424859928; __utma=48291897.1920814331.1424859928.1447757583.1447769493.109; __utmz=48291897.1445328836.98.21.utmcsr=Focus%20sur...|utmccn=La%20Lettre%20d'information%20des%20élu(e)s%20de%20l'agglomération%20nantaise%2013%20(elus)|utmcmd=e-mail"
| name:"http_host" type:string value:"toto.example.org"
| name:"http_user_agent" type:string value:"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0"
| name:"response" type:string value:"HTTP/1.1 406 Not Acceptable"
| name:"sent_http_connection" type:string value:"Keep-Alive"
| name:"sent_http_content_encoding" type:string value:"gzip"
| name:"sent_http_content_length" type:string value:"260"
| name:"sent_http_content_type" type:string value:"text/html; charset=iso-8859-1"
| name:"sent_http_keep_alive" type:string value:"timeout=5, max=98"
| name:"sent_http_vary" type:string value:"Accept-Encoding"
| name:"module_message" type:string value:"Access denied with code 406 (phase 2). [file "/usr/local/share/modsecurity-crs/2.2.5+nantes/base_rules/modsecurity_crs_41_xss_attacks.conf"] [line "765"] [id "973335"] [rev "2.2.5"] [msg "IE XSS Filters - Attack Detected"] [data "'information%20des%20\xc3\xa9lu(e)"]"
| name:"module_action" type:string value:"Intercepted (phase 2)"
| name:"module_apache_handler" type:string value:"proxy-server"
| name:"module_stopwatch" type:string value:"1448011856836563 7231 (- - -)"
| name:"module_stopwatch2" type:string value:"1448011856836563 7231; combined=6334, p1=995, p2=5286, p3=0, p4=0, p5=53, sr=58, sw=0, l=0, gc=0"
| name:"module_producer" type:string value:"ModSecurity for Apache/2.6.6 (http://www.modsecurity.org/); OWASP_CRS/2.2.5."
| name:"module_server" type:string value:"Apache"
--]]
local string = require 'string'
local table = require 'table'
local l = require 'lpeg'
l.locale(l)
local dt = require 'date_time'
local clf = require 'common_log_format'
local ip = require 'ip_address'
local msg_type = read_config('type') or 'apache.security'
local uat = read_config("user_agent_transform")
local uak = read_config("user_agent_keep")
local uac = read_config("user_agent_conditional")
local payload_keep = read_config("payload_keep")
local integer = (l.P'-'^-1 * l.digit^1) / tonumber
local ipv4 = l.Ct(l.Cg(ip.v4, 'value') * l.Cg(l.Cc'ipv4', 'representation'))
local ipv6 = l.Ct(l.Cg(ip.v6, 'value') * l.Cg(l.Cc'ipv6', 'representation'))
local ipv46 = ipv4 + ipv6
local function append(t, i, v)
i = string.gsub(string.lower(i), "-", "_")
if t[i] == nil then
t[i] = {}
end
table.insert(t[i], v)
return t
end
local header_name = l.C((l.P(1) - l.P':')^1)
local header_value = l.C((l.P(1)-l.P'\n')^0)
local header_pair = l.Cg(header_name * l.P': ' * header_value) * l.P'\n'
local header_pairs = l.Cf(l.Ct('') * header_pair^0, append)
local function header_match(a)
return header_pairs:match(a)
end
local boundary = (l.xdigit)^1
local boundary_cg = l.Cg(boundary, 'boundary')
local modsec_date_cg = l.Cg(l.Ct(
l.Cg(l.digit^-2, 'day')
* l.P'/'
* dt.date_mabbr
* l.P'/'
* dt.date_fullyear
* l.P':'
* dt.time_hour
* l.P':'
* dt.time_minute
* l.P':'
* dt.time_second
* l.P' '
* dt.timezone_offset
) / dt.time_to_ns, 'time')
local modsec_grammar_ct = l.Ct(
-- Audit Log Header
l.P'--' * boundary_cg * l.P'-A--' * l.P'\n'
* l.P'[' * modsec_date_cg * l.P'] '
* l.Cg((l.P(1)-l.P' ')^1, 'transaction_id')
* l.P' '
* l.Cg(ipv46, 'remote_addr')
* l.P' '
* l.Cg(integer, 'remote_port')
* l.P' '
* l.Cg(ipv46, 'server_addr')
* l.P' '
* l.Cg(integer, 'server_port')
* l.P'\n'
-- Request Headers
* (
l.P'--' * boundary * l.P'-B--' * l.P'\n'
* l.Cg((l.P(1) - l.P'\n')^1, 'request')
* l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^1 / header_match, 'http')
* l.P'\n'
)^-1
-- Request Body
* (
l.P'--' * boundary * l.P'-C--' * l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^0, 'request_body')
* l.P'\n'
)^-1
-- Fake Request Body
* (
l.P'--' * boundary * l.P'-I--' * l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^0, 'fake_request_body')
* l.P'\n'
)^-1
-- Response Headers
* (
l.P'--' * boundary * l.P'-F--' * l.P'\n'
* l.Cg((l.P(1) - l.P'\n')^1, 'response')
* l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^1 / header_match, 'sent_http')
* l.P'\n'
)^-1
---- Response Body
* (
l.P'--' * boundary * l.P'-E--' * l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^0, 'response_body')
* l.P'\n'
)^-1
-- Audit Log Trailer
* (
l.P'--' * boundary * l.P'-H--' * l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^1 / header_match, 'module')
* l.P'\n'
)^-1
-- Multipart Files Information
* (
l.P'--' * boundary * l.P'-J--' * l.P'\n'
* (l.P(1) - (l.P'\n--' * boundary))^1 -- FIXME
* l.P'\n'
)^-1
-- Matched Rules
* (
l.P'--' * boundary * l.P'-K--' * l.P'\n'
* l.Cg((l.P(1) - (l.P'\n--' * boundary))^0, 'matched_rules')
* l.P'\n'
)^-1
-- Audit Log Footer
* l.P'--' * boundary * l.P'-Z--' * l.P'\n'
* l.P'\n'^1
* l.P(-1)
)
local message_grammar = l.Ct(
( -- From msre_format_metadata in apache2/re.c
l.Cg((l.P(1) - l.P' [')^1, 'message')
* (l.P' [file "' * l.Cg((l.P(1) - l.P'"]')^1, 'file') * l.P'"]')^-1
* (l.P' [line "' * l.Cg((l.P(1) - l.P'"]')^1, 'line') * l.P'"]')^-1
* (l.P' [id "' * l.Cg((l.P(1) - l.P'"]')^1, 'id') * l.P'"]')^-1
* (l.P' [rev "' * l.Cg((l.P(1) - l.P'"]')^1, 'rev') * l.P'"]')^-1
* (l.P' [msg "' * l.Cg((l.P(1) - l.P'"]')^1, 'msg') * l.P'"]')^-1
* (l.P' [data "' * l.Cg((l.P(1) - l.P'"]')^1, 'data') * l.P'"]')^-1
* (l.P' [severity "' * l.Cg((l.P(1) - l.P'"]')^1, 'severity') * l.P'"]')^-1
* (l.P' [ver "' * l.Cg((l.P(1) - l.P'"]')^1, 'ver') * l.P'"]')^-1
* (l.P' [msg "' * l.Cg((l.P(1) - l.P'"]')^1, 'msg') * l.P'"]')^-1
* (l.P' [ver "' * l.Cg((l.P(1) - l.P'"]')^1, 'ver') * l.P'"]')^-1
* (l.P' [maturity "' * l.Cg((l.P(1) - l.P'"]')^1, 'maturity') * l.P'"]')^-1
* (l.P' [accuracy "' * l.Cg((l.P(1) - l.P'"]')^1, 'accuracy') * l.P'"]')^-1
* l.Cg(l.Ct((' [tag "' * lpeg.C((l.P(1) - lpeg.P'"]')^0) * l.P'"]')^0),'tag')
* l.P(-1)
) + ( -- From msre_op_validateHash_execute in apache2/re_operators.c
l.P'Rule '
* l.Cg((l.P(1) - l.P' [')^1, 'rule')
* l.P' [id "'
* l.Cg((l.P(1) - l.P'"]')^1, 'id')
* l.P'"]'
* l.P'[file "'
* l.Cg((l.P(1) - l.P'"]')^1, 'file')
* l.P'"]'
* l.P'[line "'
* l.Cg((l.P(1) - l.P'"]')^1, 'line')
* l.P'"] - '
* l.Cg(l.P(1)^1, 'message') -- 'Execution error - PCRE limits exceeded (-8): (null).'
)
)
local msg = {
Timestamp = nil,
Type = msg_type,
Hostname = nil,
Payload = nil,
Pid = nil,
Severity = nil,
Fields = nil
}
function process_message ()
local log = read_message('Payload')
local fields = modsec_grammar_ct:match(log)
if not fields then return -1 end
msg.Timestamp = fields.time
fields.time = nil
msg.Fields = {}
for k, v in pairs(fields) do
if k == 'http' or k == 'sent_http' or k == 'module' then
for k2, v2 in pairs(v) do
msg.Fields[k .. '_' .. k2] = v2
end
else
msg.Fields[k] = v
end
end
if payload_keep then
msg.Payload = log
end
if msg.Fields.http_user_agent and uat then
msg.Fields.user_agent_browser,
msg.Fields.user_agent_version,
msg.Fields.user_agent_os = clf.normalize_user_agent(msg.Fields.http_user_agent[1])
if not ((uac and not msg.Fields.user_agent_browser) or uak) then
msg.Fields.http_user_agent = nil
end
end
if msg.Fields.module_message then
local message_fields
local field_name
for i, v in ipairs(msg.Fields.module_message) do
message_fields = message_grammar:match(v)
if message_fields then
for k2, v2 in pairs(message_fields) do
field_name = 'module_message_' .. k2
if msg.Fields[field_name] == nil then
msg.Fields[field_name] = {}
end
if type(v2) == 'table' then
for i3, v3 in ipairs(v2) do
table.insert(msg.Fields[field_name], v3)
end
else
table.insert(msg.Fields[field_name], v2)
end
end
end
end
if msg.Fields.module_message_tag and #msg.Fields.module_message_tag == 0 then
msg.Fields.module_message_tag = nil
end
end
inject_message(msg)
return 0
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment