Send Googlebot requests per day by status code into Slack - For NGINX web servers

server_stats.sh

yesterdaysTimestamp=$(date -d "yesterday 13:00" +'%d/%b/%Y')
# date is expected to be passed in the following format 18/Jul/2017
siteName="example.com"
logPath="/srv/www/$siteName/logs/access.log"
# Depending on the size of the access log you might have to add * to include multiple files
# To grep through .gz files use zgrep instead of grep

# grep -v 'onpage' excludes requests from onpage bot, add pipes to exclude other unwanted bots using Googlebot as their UserAgent
# Prepare results
googleBot=$(grep 'Googlebot' $logPath | grep $yesterdaysTimestamp | grep -v 'onpage' | wc -l)
response_200=$(grep 'GET' $logPath | grep $yesterdaysTimestamp | grep -v 'onpage' | grep 'Googlebot' | grep -w 200 | wc -l)
response_301=$(grep 'GET' $logPath | grep $yesterdaysTimestamp | grep -v 'onpage' | grep 'Googlebot' | grep -w 301 | wc -l)
response_404=$(grep 'GET' $logPath | grep $yesterdaysTimestamp | grep -v 'onpage' | grep 'Googlebot' | grep -w 404 | wc -l)
response_500=$(grep 'GET' $logPath | grep $yesterdaysTimestamp | grep -v 'onpage' | grep 'Googlebot' | grep -w 500 | wc -l)

# Send to Slack webhook, change the webhook URL
curl -X POST -H 'Content-type: application/json' \
      --data "{ \"attachments\": [ { \"title\":\"Server stats :robot_face: - $yesterdaysTimestamp \n For $siteName \", \"text\":\"Googlebot total requests - *$googleBot* \n _status code 200 - $response_200 _ \n _status code 301 - $response_301 _ \n _status code 404 - $response_404 _ \n _status code 500 - $response_500 _ \", \"mrkdwn_in\": [ \"text\"] } ] }" \
      https://hooks.slack.com/services/XXXXXXXX/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXX

# Configure as cronjob, to get daily stats into Slack