Common headers

count_headers.awk

BEGIN {
    st_waitnewconn = 0
    st_headers = 1
    st_body = 2

    size_threshold = 4000
}

function count_headers(headers_string, dir) {
    split(headers_string, headers, "\n")
    for (i in headers) {
        split(headers[i], header_parts, ": ")
        if (length(header_parts) >= 2 && tolower(header_parts[1]) == "cookie") {
            split(header_parts[2], cookies, "; ")
            for (ci in cookies) {
                split(cookies[ci], cookie_part, "=")
                cookie_counts[cookie_part[1]] += dir
            }

            #header_counts[header_parts[1]]++
        }
    }
}

/^T .* -> .* \[AP\]/ {
    current_host = $2
    if (host_states[current_host] == "") {
        host_states[current_host] = st_waitnewconn
    }
    #print("host is now", current_host, "state", host_states[current_host])
    next
}

/^\.$/ && host_states[current_host] == st_headers {
    host_states[current_host] = st_body
    #print("host is now", current_host, "state", host_states[current_host])
    #print("acc headers", headers_buffer[current_host])

    headers_string = headers_buffer[current_host]
    headers_len = length(headers_string)

    if (headers_len > size_threshold) {
        count_headers(headers_string, 1)
        print large_count++ > "/dev/stderr"
    }

    if (large_count > 100) {
        exit
    }

    next
}

/^[A-Z]+ \/.* HTTP\/.*/ && (host_states[current_host] == st_waitnewconn || host_states[current_host] == st_body) {
    host_states[current_host] = st_headers
    headers_buffer[current_host] = ""
    next
}

/^$/ {
    current_host = ""
    next
}

current_host != "" && host_states[current_host] == st_headers {
    headers_buffer[current_host] = headers_buffer[current_host] $0 "\n"
}

END {
    for (name in cookie_counts) {
        print cookie_counts[name], name
    }
}

gistfile1.txt

sudo ngrep -p -d ens5 -W byline -q tcp and port 80 and dst port 80 |
  gawk -f count_headers.awk |
  grep -v _gac_UA- |
  sort -n