Fetch, Merge, and Strip Blocklists from url list

update-blocklists.sh

#!/bin/bash 

# Sync Blocklists.
# 	by boTux.
# inspired by : 
# 	https://gist.github.com/raidzero/9202581#file-blocklist-update-sh
# 	https://gist.github.com/jult/e76c628899bd5aa3c33a

# Add cron :
# tout les jours a 05:00
# 0 5 * * * ~/.blocklists/update-blocklists.sh

# Config
BLOCKLISTS_HOME=~/.blocklists
BLOCKLISTS_EXPORT=~/.blocklists/export
BLOCKLISTS_TMP=~/.blocklists/tmp
BLOCKLISTS_DL=~/.blocklists/tmp/download

# Initialiser
cd $BLOCKLISTS_HOME

# [1] Simple AutoDownload of all Bluetack lists.
#################################################
# curl -s https://www.iblocklist.com/lists.php | grep -A 2 Bluetack | xargs wget -qO -  | gunzip -f | egrep -v '^#' > $BLOCKLISTS_HOME/cumul.blocklists

# [2] Personnalized list to download and convert.
#################################################
# To add other iblocklists : Just add URL in the list - Only with gz !

# Generating blocklists list :
cat << END_LIST > $BLOCKLISTS_TMP/blocklists
http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_level2&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_level3&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_edu&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_templist&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=bt_hijacked&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=ijfqtofzixtwayqovmxn&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=ecqbsykllnadihkdirsh&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=jcjfaxgyyshvdbceroxf&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=lljggjrpmefcwqknpalp&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=pfefqteoxlfzopecdtyw&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=tbnuqfclfkemqivekikv&fileformat=p2p&archiveformat=gz
http://list.iblocklist.com/?list=ewqglwibdgjttwttrinl&fileformat=p2p&archiveformat=gz
http://www.wael.name/wael.list.p2p.gz
http://john.bitsurge.net/public/biglist.p2p.gz
END_LIST

mkdir -p $BLOCKLISTS_DL
cd $BLOCKLISTS_DL
rm -rf $BLOCKLISTS_DL/*
echo "Removed existing blocklists."

# Downloading
NUMBER=1
echo -n "Downloading blocklist "
while read URL; do 
  echo -n "$NUMBER..."
  wget -q $URL
  if [ $? -ne 0 ]; then
	echo "$NUMBER FAILED!"
  fi
  let NUMBER=$NUMBER+1
done < $BLOCKLISTS_TMP/blocklists
echo " --> Done!"
# Unpacking
NUMBER=1
echo -n "Unpacking blockist "
for file in `find . -type f`; do 
  echo -n "$NUMBER..."
  mv $file $file.gz
  gunzip $file.gz
  # Remove comment and blank lines, so transmission doesnt yell about invalid lines
  sed -i '/\#.*$/d;/^$/d' $file
  let NUMBER=$NUMBER+1
done
echo " --> Done!"
# Renaming
NUMBER=1
for file in `find . -type f -name "*gz"`; do
  mv $file $NUMBER-blocklist
  let NUMBER=$NUMBER+1
done
# Cumul list into one file cumul.blocklists
NUMBER=1
echo -n "Merging blockists "
for file in `find . -type f -name "*-blocklist"`; do
  echo -n "$NUMBER..."
  cat $file >> $BLOCKLISTS_DL/cumul.blocklists
  # Remove single files
  rm $file
  let NUMBER=$NUMBER+1
done
echo " --> Done!"

#clean up the list
rm $BLOCKLISTS_TMP/blocklists

# Move the list to home
mv $BLOCKLISTS_DL/cumul.blocklists $BLOCKLISTS_TMP/cumul.blocklists
# Remove olds exports
rm $BLOCKLISTS_EXPORT/*
# Strip, Sort and Delete doubles:
echo -n "[1] Strip blockists "
sed -i "s/[[:space:]]*#.*$//g" $BLOCKLISTS_TMP/cumul.blocklists
sed -i "s/.*value='\(http:.*\)'.*/\1/p" $BLOCKLISTS_TMP/cumul.blocklists
sort $BLOCKLISTS_TMP/cumul.blocklists | uniq -u > $BLOCKLISTS_EXPORT/blocklist-full.txt
rm $BLOCKLISTS_TMP/cumul.blocklists
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-full.txt"

# Strip everything but IPv4s (and ranges), for use in firewalls etc.
echo -n "[2] Extract IPV4 "
grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}$|([0-9]{1,3}\.){3}[0-9]{1,3}\-([0-9]{1,3}\.){3}[0-9]{1,3}' $BLOCKLISTS_EXPORT/blocklist-full.txt > $BLOCKLISTS_TMP/ipv4.blocklists
sort $BLOCKLISTS_TMP/ipv4.blocklists | uniq -u > $BLOCKLISTS_EXPORT/blocklist-ipv4.txt
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4.txt"

# Beautyfi for tixati pulling, add colon in front
echo -n "[3] Beautyfi IPV4 "
sed "s/^/:/g" $BLOCKLISTS_EXPORT/blocklist-ipv4.txt > $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt"

# Keep as txt and gz file:
echo -n "[4] Gzip all blocklists"
gzip -c $BLOCKLISTS_EXPORT/blocklist-full.txt > $BLOCKLISTS_EXPORT/blocklist-full.txt.gz
gzip -c $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt > $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt.gz
gzip -c $BLOCKLISTS_EXPORT/blocklist-ipv4.txt > $BLOCKLISTS_EXPORT/blocklist-ipv4.txt.gz
echo " --> Done!"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4.txt.gz"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-ipv4-print.txt.gz"
echo " +--[Exported]--> $BLOCKLISTS_EXPORT/blocklist-full.txt.gz"
echo ""
# Let's make sure next run is a clean one
rm -rf $BLOCKLISTS_TMP/*

# Quick Count
echo -n "Total IPs in the db :"
AMOUNT=`cat $BLOCKLISTS_EXPORT/blocklist-ipv4.txt | wc -l | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta'`
echo "           $AMOUNT ip added in blocklist-ipv4.txt"

# Quit message
echo ""
echo "[Usage]"
echo "In transmission add :"
echo ""
echo "           file://$BLOCKLISTS_EXPORT/blocklist-ipv4.txt.gz"
echo ""
echo "by boTux."
exit 0

mkdir -p $BLOCKLISTS_EXPORT missing at line 50. Otherwise script doesn't work out of the box.

And probably change quit message, because Transmission 2.94 doesn't support 'file://' links in blocklists settings. To use compiled blocklist, just place blocklist-ipv4-print.txt file into ~/.config/transmission/blocklists directory.