LUA script for Suricata to check for Hook-like websocket packets

hook.lua

--[[
    Author: FOX-SRT
    created_at: 2023-06-02
    updated_at: 2023-06-07
    revision: 2

    Script to check for Hook-like websocket packets.

    For a websocket packet, the first two bytes of the TCP payload are part of the Websocket header.
    The next 4 bytes denote a XOR key that mask the remainder of the payload.

    First, grab the 4 bytes of the XOR key. Then, decode the remainder of the payload by applying the XOR key.

    Finally, this rule matches on two parts:
        - The decoded payload should begin with 42["login",
        - The decoded payload should end with '\n"]'

    In currently found samples, Hook has an encrypted payload between these parts that is also encoded using BASE64.
    Therefore, an additional rule condition could be to verify if the string between the two aforementioned matches is
    valid base64. Since no false positives were encountered during testing of this rule, this has not been included.

]]

function init (args)
    local needs = {}
    needs["payload"] = tostring(true)
    return needs
end

-- Decode the payload by applying the XOR key 'key' (assumed 4 bytes)
function unmask_payload(payload, key)
    local tohex = bit.tohex
    local unmasked = {}
    unmasked = ""
    -- The 'payload' variable contains the full TCP payload. 
    -- Skip 2 bytes for the Websocket header, 4 bytes for the encryption key
    for i=7,#payload,1
    do
        xor_key_current_index = (i - 7) % 4 + 1
        xor_key_character = string.byte(key, xor_key_current_index)
        unmasked_character = string.char(bit.bxor(string.byte(payload,i),xor_key_character))
        unmasked = unmasked .. unmasked_character
    end
    return unmasked
end

-- Simple helper functions
function string.starts(String,Start)
    return string.sub(String,1,string.len(Start))==Start
 end

 function string.ends(String,End)
    ending = string.sub(String,-#End)
    return ending==End
 end

function match(args)
    local payload = tostring(args["payload"])
    if #payload > 0 then
        local key = string.sub(payload,3,6)
        local unmasked_payload = unmask_payload(payload, key)

        -- Do Hook specific matching on the unmasked websocket payload
        if string.starts(unmasked_payload, '42["login",') then
            if string.ends(unmasked_payload, '\\n"]') then
                return 1
            end
        end
        return 0
    end
return 0
end

Blog post: https://blog.fox-it.com/2023/09/11/from-ermac-to-hook-investigating-the-technical-differences-between-two-android-malware-variants/