Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
What to do with apache logs?
#! /bin/bash
# only work on files older then $days
days=7
# look for files in this directory
logDir="/var/log/apache2/"
# with this extension
extension="*.gz"
# depersonalize by replacing the last two bytes of an IP with this string
anoBytes=".0.0"
# define how to behave after anonymizing
function postProcess(){
# echo remove $1
rm -f $1
}
#-----------config end-----------------
# Generate list of files to work on
filesToAnnonymize=`find $logDir -type f -name "$extension" ! -name "*.ano.*" -mtime +$days`
#echo "Try to annonymize $filesToAnnonymize"
# Test if
if touch $logDir/.annonymize-apache-logs.sh.test > /dev/null 2>&1
then
rm $logDir/.annonymize-apache-logs.sh.test
else
echo "You don't have permission to write into $logDir"
exit 1;
fi
#Work through each file
for file in `echo $filesToAnnonymize`
do
datestamp=`date +"%Y%m%d%H%M%s"`
# echo Process $file
zcat $file |sed -E "s/([0-9]{1,3}\.[0-9]{1,3})\.[0-9]{1,3}\.[0-9]{1,3}/\1$anoBytes/"|gzip > ${file%.*}.ano.${datestamp}.gz
postProcess $file
done
#! /bin/bash
YESTERDAY=$(LANG=en;date +"%d/%b/%Y" -d"yesterday")
STATS_LOG=stats.log
MATOMO_URL=https://your.matomo.webapi
MATOMO=/dir/with/matomo/installation
MATOMO_ADMIN=admin
MATOMO_PASSWORD=yourpassword
IDSITE=4
PREFIX=/var/log/apache2/localhost-access.*
function loadLogFile() {
PATTERN=$1
echo Lade $PATTERN >> $STATS_LOG
zgrep --no-filename $PATTERN /var/log/apache2/other_vhosts_access.*|sort|uniq > /tmp/matomoImport.log
python $MATOMO/misc/log-analytics/import_logs.py --recorder-max-payload-size=200 --url $MATOMO_URL --login $MATOMO_ADMIN --password $MATOMO_PASSWORD /tmp/matomoImport.log --idsite=$IDSITE >> $STATS_LOG
}
loadLogFile "$YESTERDAY"
# Beispiel um die Logfiles der letzten 10-40 Tage zu laden
#
#for i in {10..40};do loadLogFile "$(LANG=en;date +"%d/%b/%Y" -d"$i days ago")";done
# Beispiel um die Differenz in Tagen auszurechnen
#
# A="2018-01-21"
# B="2018-04-08"
# echo $(( (`date -d $B +%s` - `date -d $A +%s`) / 86400 )) days
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.