access_unique_ips.sh

#! /usr/bin/env bash

rm -rf logs
mkdir logs

all_log_files=""
for server in "germany" "helsinki" "us-east" "us-west"; 
do
    scp -rpq user@${server}.siasky.net:skynet-webportal/docker/data/nginx/logs logs/${server}
    for file in logs/${server}/*.gz; do gunzip -d ${file}; done
    rm -rf logs/${server}/*.gz
    all_log_files="${all_log_files} $(ls -d logs/${server}/access.log*)"
done

####################################################################
####################################################################

healthcheck=AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q

####################################################################
####################################################################

row_template="%-20s %14s %14s %14s\n"
row_separator="=================================================================\n"
section_separator="\n\n"

####################################################################
####################################################################

printf "${row_template}" "UNIQUE IPS" "Uploads" "Downloads" "Total"
printf "${row_separator}"
for month in "Mar/2020" "Apr/2020" "May/2020" "Jun/2020" "Jul/2020";
do 
    uploads=$(cat ${all_log_files} | grep -E ${month} | grep -vE ${healthcheck} | grep -E 'POST ' | awk '{ print $1 }' | sort | uniq -c | sort -n | wc -l)
    downloads=$(cat ${all_log_files} | grep -E ${month} | grep -vE ${healthcheck} | grep -E 'GET ' | awk '{ print $1 }' | sort | uniq -c | sort -n | wc -l)
    printf "${row_template}" ${month} ${uploads} ${downloads} $((uploads + downloads))
done
printf "${row_separator}"
total_unique_uploads=$(cat ${all_log_files} | grep -E 'POST ' | grep -vE ${healthcheck} | awk '{ print $1 }' | sort | uniq -c | sort -n | wc -l)
total_unique_downloads=$(cat ${all_log_files} | grep -E 'GET ' | grep -vE ${healthcheck} | awk '{ print $1 }' | sort | uniq -c | sort -n | wc -l)
printf "${row_template}" "Total" ${total_unique_uploads} ${total_unique_downloads} $((total_unique_uploads + total_unique_downloads))

####################################################################
printf "${section_separator}"
####################################################################

printf "${row_template}" "ALL REQUESTS" "Uploads" "Downloads" "Total"
printf "${row_separator}"
for month in "Mar/2020" "Apr/2020" "May/2020" "Jun/2020" "Jul/2020";
do 
    uploads=$(cat ${all_log_files} | grep -E ${month} | grep -vE ${healthcheck} | grep -E 'POST ' | wc -l)
    downloads=$(cat ${all_log_files} | grep -E ${month} | grep -vE ${healthcheck} | grep -E 'GET ' | wc -l)
    printf "${row_template}" ${month} ${uploads} ${downloads} $((uploads + downloads))
done
printf "${row_separator}"
total_uploads=$(cat ${all_log_files} | grep -vE ${healthcheck} | grep -E 'POST ' | grep -v ' 172.' | grep -v ' 192.168.' | wc -l)
total_downloads=$(cat ${all_log_files} | grep -vE ${healthcheck} | grep -E 'GET ' | grep -v ' 172.' | grep -v ' 192.168.' | wc -l)
printf "${row_template}" "Total" ${total_uploads} ${total_downloads} $((total_uploads + total_downloads))

####################################################################
####################################################################