Find interesting referers in access.log - see https://jay.gooby.org/2021/11/30/find-interesting-referers-in-access-log for more details

interesting-referers

#!/bin/bash
#
# MIT License
#
# Copyright (c) 2021 Jay Caines-Gooby, @jaygooby, jay@gooby.org
#
# The one-liner from https://jay.gooby.org/2021/11/30/find-interesting-referers-in-access-log
# 
# Usage: interesting-referers /path/to/access.log
#        interesting-referers /path/to/access.log example.com
# 
# The first usage example defaults to filtering out any
# referer lines with your host in, the second example
# lets you override that host and provide another to ignore
#
# Edit the jay.gooby.org below to reflect your own web host
log="$1"
host=${2:-jay.gooby.org}

# Leave the rest as is
[ -z "$log" ] && echo "Usage: $(basename "$0") /path/to/access.log [hostname]" >&2 && exit 1

zcat -f "$log" | awk -v host=$host '{if ($7 !~ /\.[^html]/ && $11 != "\"-\"" && $11 !~ host) {print $7 "\t" $11}}' | sort | uniq -c | sort