Performs reverse and forward DNS lookups to list Googlebot's IPs, given a list of IP addresses as a file. Useful for filtering access logs to find out actual Googlebot visits. An implementation of https://support.google.com/webmasters/answer/80553?hl=en

filter-googlebot-ips.sh

#/bin/bash
#
# Performs reverse and forward DNS lookups to list Googlebot's IPs, given a list
# of IP addresses as a file. Useful for filtering access logs to find out actual
# Googlebot visits.
#
# An implementation of https://support.google.com/webmasters/answer/80553?hl=en

while IFS='' read -r IP_ADDRESS || [[ -n "$IP_ADDRESS" ]];
do
    IS_GOOGLEBOT=0
    REVERSE_LOOKUP="$(host $IP_ADDRESS)"
    echo "$REVERSE_LOOKUP" | grep -E "google.com.$|googlebot.com.$" > /dev/null && IS_GOOGLEBOT=1

    if [[ IS_GOOGLEBOT -eq 1 ]]; then
        FORWARD_LOOKUP="$(host $(echo "$REVERSE_LOOKUP" | cut -d " " -f 5) | cut -d " " -f 4)"
        if [[ "$FORWARD_LOOKUP" = "$IP_ADDRESS" ]];
        then
            echo $IP_ADDRESS
        fi
    fi
done < "$1"