websocket.lua

local pack, unpack, char, byte = string.pack, string.unpack, string.char, string.byte
local random = math.random

local leftover_fmt = {'>I1', '>I2', '>I3'}

local function mask_message(data, mask)
    local res = {}

    local len = #data
    local left = len & 3;
    local j = 1

    for i = 1, len - 3, 4 do
        res[j] = pack('>I4', unpack('>I4', data, i) ~ mask)
        j = j + 1
    end

    if left ~= 0 then
        local leftover_mask = mask >> (8 * (4 - left))
        res[j] = pack(leftover_fmt[left], unpack(leftover_fmt[left], data, len - left + 1) ~ leftover_mask)
    end

    return table.concat(res)
end

-- RAND_MAX has a minimum value of 0x7fff, so assume that's what we have
local function rand32()
    return random(0x1000, 0xffff) << 16 | random(0x1000, 0xffff)
end

local function encode(opcode, payload, options)
    options = options or {}
    local msg = {}

    msg[1] = opcode
    msg[1] = msg[1] | (options.fin ~= false and 0x80 or 0) -- fin is true by default
    msg[1] = msg[1] | (options.rsv1 == true and 0x40 or 0)
    msg[1] = msg[1] | (options.rsv2 == true and 0x20 or 0)
    msg[1] = msg[1] | (options.rsv3 == true and 0x10 or 0)
    msg[1] = char(msg[1])

    local len = #payload

    if len < 126 then
        msg[2] = char(len | (options.mask and 0x80 or 0))
    elseif len < 0x10000 then
        msg[2] = pack('>BI4', 126 | (options.mask and 0x80 or 0), len)
    else
        msg[2] = pack('>BI8', 127 | (options.mask and 0x80 or 0), len)
    end

    if options.mask then
        local mask = rand32()
        msg[3] = pack('>I4', mask)
        msg[4] = mask_message(payload, mask)
    else
        msg[3] = payload
    end

    return table.concat(msg)
end

local function decode(message)
    local options = {}

    local opcode = byte(message, 1) & 0xf
    options.fin = byte(message, 1) & 0x80 ~= 0
    options.rsv1 = byte(message, 1) & 0x40 ~= 0
    options.rsv2 = byte(message, 1) & 0x20 ~= 0
    options.rsv3 = byte(message, 1) & 0x10 ~= 0

    local len = byte(message, 2) & 0x7f
    local offset = 3

    if len == 126 then
        len = unpack('>I2', message, offset)
        offset = 5
    elseif len == 127 then
        len = unpack('>I8', message, offset)
        offset = 11
    end

    if byte(message, 2) & 0x80 ~= 0 then
        options.mask = unpack('>I4', message, offset)
        offset = offset + 4
    end

    local payload = message:sub(offset, offset + len - 1)
    if options.mask then
        payload = mask_message(payload, options.mask)
    end

    return opcode, payload, options
end

local encoded = encode(0x1, 'Hello, world!', {mask = true})
local op, decoded, options = decode(encoded)

print('opcode: ' .. op)
print('payload: ' .. decoded)
print('fin: ' .. tostring(options.fin))
print('rsv1: ' .. tostring(options.rsv1))
print('rsv2: ' .. tostring(options.rsv2))
print('rsv3: ' .. tostring(options.rsv3))
print('mask: ' .. tostring(options.mask))