web server log analysis cheat sheet

# get total requests by status code
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

# get top requesters by IP
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top requesters by user agent
awk -F'"' '{print $6}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top requests by URL
awk '{print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top IP addresses requesting non-existent content
awk '$9 ~ /404/ {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top URL returning 404 Not Found
awk '$9 ~ /404/ {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top user agents requesting non-existent content
awk '$9 ~ /404/' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head

# get top IP addresses causing backend errors
awk '$0 ~ /\[error\]/ && match($0, /(client: )(.*)(, server)/, arr) {print arr[2]}' /var/log/nginx/error.log | sort | uniq -c | sort -rn

# get slower requests by URL (ignoring requests using POST method)
awk -F'rt=' '$0 !~ /POST/ && substr($2,0,5) > 5' /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head