Instantly share code, notes, and snippets.

Embed
What would you like to do?
Create blacklist for HAVP from phishtank and adserver blacklists
#!/bin/sh
cd /etc/havp
OUTFILE=/etc/havp/blacklist
ADSERVERLIST=/etc/havp/adserverlist
PHISHTANK=/etc/havp/phishtank
MYBLACKLIST=/etc/havp/myblacklist
wget -q -N "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=webwasher;showintro=0;mimetype=plaintext"
sed -e 's_^//_#_g' serverlist.php* | sort | uniq > $ADSERVERLIST
wget -q -N http://data.phishtank.com/data/<PhishTank API key>/online-valid.csv.bz2
bzcat online-valid.csv.bz2|sed -e 's/^[0-9]*,//' -e 's@,http://www.phishtank.com/phish_detail.php?phish_id=[0-9]*,.*$@@' -e 's/^"\(.*\)"$/\1/' -e 's_^https\?://__' -e 's/#.*//' -e 's_/$_/*_' -e 's_^\([^/]*\)$_\1/*_' -e 's/?.*/*/' -e 's_^\([^/]*\)\*$_\1/*_' -e 's/\*.*/*/'| grep -vF 'phish_id,url,phish_detail_url,submission_time,verified,verification_time,online,target' | iconv -f utf8 -t ascii -c - | sort | uniq > $PHISHTANK
echo "# blacklist file generated by $0, `date`" > $OUTFILE
echo "\n# MYBLACKLIST:" >> $OUTFILE
cat $MYBLACKLIST >> $OUTFILE
echo "\n# ADSERVERLIST:" >> $OUTFILE
cat $ADSERVERLIST >> $OUTFILE
echo "\n# PHISHTANK:" >> $OUTFILE
cat $PHISHTANK >> $OUTFILE
@ovidiubica

This comment has been minimized.

ovidiubica commented Mar 2, 2016

Any idea why it can't read the blacklist upon restart? I even tried to chown havp:havp /etc/havp/blacklist

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment