Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Fetch, Merge, and Strip Blocklists from url list
#!/bin/bash
# Sync Blocklists.
# by boTux.
# inspired by :
# https://gist.github.com/raidzero/9202581#file-blocklist-update-sh
# https://gist.github.com/jult/e76c628899bd5aa3c33a
# Add cron :
# tout les jours a 05:00
# 0 5 * * * ~/.blocklists/update-blocklists.sh
# Config
BLOCKLISTS_HOME=~/.blocklists
BLOCKLISTS_EXPORT=~/.blocklists/export
BLOCKLISTS_TMP=~/.blocklists/tmp
BLOCKLISTS_DL=~/.blocklists/tmp/download
# Initialiser
cd $BLOCKLISTS_HOME
# [1] Simple AutoDownload of all Bluetack lists.
#################################################
# curl -s https://www.iblocklist.com/lists.php | grep -A 2 Bluetack | xargs wget -qO - | gunzip -f | egrep -v '^#' > $BLOCKLISTS_HOME/cumul.blocklists
# [2] Personnalized list to download and convert.
#################################################
# To add other iblocklists : Just add URL in the list - Only with gz !
# Generating blocklists list :
cat << END_LIST > $BLOCKLISTS_TMP/blocklists
http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_level2&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_level3&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_edu&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_templist&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_hijacked&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=ijfqtofzixtwayqovmxn&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=ecqbsykllnadihkdirsh&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=jcjfaxgyyshvdbceroxf&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=lljggjrpmefcwqknpalp&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=pfefqteoxlfzopecdtyw&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=tbnuqfclfkemqivekikv&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=ewqglwibdgjttwttrinl&fileformat=p2p&archiveformat=gz
http://www.wael.name/wael.list.p2p.gz
http://john.bitsurge.net/public/biglist.p2p.gz
END_LIST
mkdir -p $BLOCKLISTS_DL
cd $BLOCKLISTS_DL
rm -rf $BLOCKLISTS_DL/*
echo "Removed existing blocklists."
# Downloading
NUMBER=1
echo -n "Downloading blocklist "
while read URL; do
echo -n "$NUMBER..."
wget -q $URL
if [ $? -ne 0 ]; then
echo "$NUMBER FAILED!"
fi
let NUMBER=$NUMBER+1
done < $BLOCKLISTS_TMP/blocklists
echo " --> Done!"
# Unpacking
NUMBER=1
echo -n "Unpacking blockist "
for file in `find . -type f`; do
echo -n "$NUMBER..."
mv $file $file.gz
gunzip $file.gz
# Remove comment and blank lines, so transmission doesnt yell about invalid lines
sed -i '/\#.*$/d;/^$/d' $file
let NUMBER=$NUMBER+1
done
echo " --> Done!"
# Renaming
NUMBER=1
for file in `find . -type f -name "*gz"`; do
mv $file $NUMBER-blocklist
let NUMBER=$NUMBER+1
done
# Cumul list into one file cumul.blocklists
NUMBER=1
echo -n "Merging blockists "
for file in `find . -type f -name "*-blocklist"`; do
echo -n "$NUMBER..."
cat $file >> $BLOCKLISTS_DL/cumul.blocklists
# Remove single files
rm $file
let NUMBER=$NUMBER+1
done
echo " --> Done!"
#clean up the list
rm $BLOCKLISTS_TMP/blocklists
# Move the list to home
mv $BLOCKLISTS_DL/cumul.blocklists $BLOCKLISTS_TMP/cumul.blocklists
# Remove olds exports
rm $BLOCKLISTS_EXPORT/*
# Strip, Sort and Delete doubles:
echo -n "[1] Strip blockists "
sed -i "s/[[:space:]]*#.*$//g" $BLOCKLISTS_TMP/cumul.blocklists
sed -i "s/.*value='\(http:.*\)'.*/\1/p" $BLOCKLISTS_TMP/cumul.blocklists
sort $BLOCKLISTS_TMP/cumul.blocklists | uniq -u > $BLOCKLISTS_EXPORT/blocklist-full.txt
rm $BLOCKLISTS_TMP/cumul.blocklists
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-full.txt"
# Strip everything but IPv4s (and ranges), for use in firewalls etc.
echo -n "[2] Extract IPV4 "
grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}$|([0-9]{1,3}\.){3}[0-9]{1,3}\-([0-9]{1,3}\.){3}[0-9]{1,3}' $BLOCKLISTS_EXPORT/blocklist-full.txt > $BLOCKLISTS_TMP/ipv4.blocklists
sort $BLOCKLISTS_TMP/ipv4.blocklists | uniq -u > $BLOCKLISTS_EXPORT/blocklist-ipv4.txt
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4.txt"
# Beautyfi for tixati pulling, add colon in front
echo -n "[3] Beautyfi IPV4 "
sed "s/^/:/g" $BLOCKLISTS_EXPORT/blocklist-ipv4.txt > $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt"
# Keep as txt and gz file:
echo -n "[4] Gzip all blocklists"
gzip -c $BLOCKLISTS_EXPORT/blocklist-full.txt > $BLOCKLISTS_EXPORT/blocklist-full.txt.gz
gzip -c $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt > $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt.gz
gzip -c $BLOCKLISTS_EXPORT/blocklist-ipv4.txt > $BLOCKLISTS_EXPORT/blocklist-ipv4.txt.gz
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4.txt.gz"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt.gz"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-full.txt.gz"
echo ""
# Let's make sure next run is a clean one
rm -rf $BLOCKLISTS_TMP/*
# Quick Count
echo -n "Total IPs in the db :"
AMOUNT=`cat $BLOCKLISTS_EXPORT/blocklist-ipv4.txt | wc -l | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta'`
echo " $AMOUNT ip added in blocklist-ipv4.txt"
# Quit message
echo ""
echo "[Usage]"
echo "In transmission add :"
echo ""
echo " file://$BLOCKLISTS_EXPORT/blocklist-ipv4.txt.gz"
echo ""
echo "by boTux."
exit 0
@AndreySV

This comment has been minimized.

Copy link

@AndreySV AndreySV commented Apr 11, 2020

mkdir -p $BLOCKLISTS_EXPORT missing at line 50. Otherwise script doesn't work out of the box.

@AndreySV

This comment has been minimized.

Copy link

@AndreySV AndreySV commented Apr 11, 2020

And probably change quit message, because Transmission 2.94 doesn't support 'file://' links in blocklists settings. To use compiled blocklist, just place blocklist-ipv4-print.txt file into ~/.config/transmission/blocklists directory.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment