web server log analysis cheat sheet

# get total requests by status code
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

# get top requesters by IP
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top requesters by user agent
cut -d' ' -f12- /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top requests by URL
awk '{print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top IP addresses requesting non-existent content
awk '($9 ~ /404/) {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top URL returning 404 Not Found
awk '($9 ~ /404/) {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# get top user agents requesting non-existent content
awk '($9 ~ /404/)' /var/log//nginx/access.log | cut -d' ' -f12- | sort | uniq -c | sort -rn | head