black/white domain list leech for dnsmasq

dnsmasqc

#!/bin/sh
alias wget=/opt/bin/wget
alias base64=/opt/bin/base64

WAN_DNS="`nvram get wan_get_dns`"
[ -z "$WAN_DNS" ] && WAN_DNS="`nvram get wan_dns`"
echo $WAN_DNS | grep " " >/dev/null 2>&1
if [ $? = 0 ];then
        FORWARDDNSISP=$(echo $WAN_DNS | awk '{print $1}')
else
        FORWARDDNSISP=$WAN_DNS
fi
FORWARDDNSISP="127.0.0.1#5353"
FORWARDDNS="127.0.0.1#5454"
SHR=0

lazysmarthosts() {
        local RD=/tmp/
        local LD=/opt/etc/dnsmasq/hosts/
        cd /tmp
        logger "Getting New SmartHosts Files..."
        #wget -qO - http://smarthosts.googlecode.com/svn/trunk/hosts > /tmp/smarthosts.temphosts
        wget -4q http://huhamhire-hosts.googlecode.com/files/ipv4_unix_utf8.zip
        unzip -qo ipv4_unix_utf8.zip
        rm ipv4_unix_utf8.zip
        mv hosts smarthosts.temphosts
        if [ -s ${RD}smarthosts.temphosts ]
        then
            rm -f ${LD}smarthosts.txt
            #sed -i -n '/#Google Services END/q;p' ${RD}smarthosts.temphosts
            sed -i -n '/# region google/,/# endregion/p' ${RD}smarthosts.temphosts
            local TMPLISTFILESIZE=$(stat -c '%s' ${RD}smarthosts.temphosts)
            if [ $TMPLISTFILESIZE -lt 1000 ] ; then
                echo "Smarthosts not retrieved or cut failed, exiting."
                SHR=0
                exit
            fi
            mv -f ${RD}smarthosts.temphosts ${LD}smarthosts_google.txt
            SHR=1
            logger "Smarthosts renew Done."
        else
            logger "Error! Smarthosts can't renew!."
        fi
}

adhosts() {
    echo "#" >/tmp/hosts
    echo "# lazy adblock" >>/tmp/hosts
    wget -qO - http://adblock-chinalist.googlecode.com/svn/trunk/adblock-lazy.txt | grep ^\|\|[^\*][^\/]*\^$ | sed -e 's:||::' -e 's:\^::' | awk '{print "0.0.0.0  " $1}' > /tmp/hosts
    echo "# lazy adblock end" >>/tmp/hosts
    echo "#" >>/tmp/hosts
    echo "# mvps adblock" >>/tmp/hosts
    wget -qO- http://winhelp2002.mvps.org/hosts.txt | tr -d '\r' | grep "^127.0.0.1" | grep -v localhost | awk '{print "0.0.0.0  " $2}' >>/tmp/hosts
    echo "# mvps adblock end" >>/tmp/hosts
    echo "#" >>/tmp/hosts
    echo "# malwaredomainlist adblock" >>/tmp/hosts
    wget -qO- http://www.malwaredomainlist.com/hostslist/hosts.txt | tr -d '\r' | grep "^127.0.0.1" | grep -v localhost | awk '{print "0.0.0.0  " $2}' >>/tmp/hosts
    echo "# malwaredomainlist adblock end" >>/tmp/hosts
    echo "#" >>/tmp/hosts
    echo "# mwsl adblock" >>/tmp/hosts
    wget -qO- http://www.mwsl.org.cn/hosts/hosts | tr -d '\r' | grep "^50.118.116.68" | grep -v localhost | awk '{print "0.0.0.0  " $2}' >>/tmp/hosts
    echo "# mwsl adblock end" >>/tmp/hosts
}

# Tunlr DNS updater for dnsmasq. 2013, Alexander Ryzhov
# Adapted For asuswrt-merlin firmware.
tunlr() {
    echo "Tunlr DNS updater started"

    local DNSMASQ_CONF='/opt/etc/dnsmasq/custom/tunlr.cfg'
    local DOMAINS=$(cat /opt/etc/dnsmasq/tunlr/domains.txt)
    local IPS=$(wget -q -O - \
        "http://tunlr.net/tunapi.php?action=getdns&version=1&format=json" \
        | sed "s/\"dns.\"://g" | sed "s/[{}]//g" | sed "s/,/\ /g" |sed "s/\"//g")
    if [ -z "$IPS" ] || [ -n "$(echo $IPS | sed 's/[0-9\.\ ]//g')" ] ; then
        echo "Tunlr DNS addresses not retrieved, exiting."
        exit
    fi
    echo -n > $DNSMASQ_CONF
    echo "# Tunlr DNS updater for dnsmasq" >> $DNSMASQ_CONF
    for domain in $DOMAINS
    do
        for dns in $IPS
        do
        echo "server=/${domain}/${dns}" >> $DNSMASQ_CONF
        done
    done
    SHR=1
    echo 'done.'
}

gfwlist_py() {
    cd /tmp
    local TMPLISTFILE="/tmp/gfwlist.txt"
    local GFWLISTFILE="/opt/etc/dnsmasq/custom/gfwlist.cfg"
    wget -q --no-check-certificate -O $TMPLISTFILE https://autoproxy-gfwlist.googlecode.com/svn/trunk/gfwlist.txt
    base64 -d $TMPLISTFILE > gfwlist_dec.txt
    sed -i '1,20d' gfwlist_dec.txt
    local TMPLISTFILESIZE=$(stat -c '%s' /tmp/gfwlist_dec.txt)
    if [ $TMPLISTFILESIZE -lt 1000 ] ; then
        echo "GFWList not retrieved or decode failed, exiting."
        SHR=0
        exit
    fi
    echo 'Converting gfwlist to domain list...'
    echo '##Anti DNS Cache Pollution##' > $GFWLISTFILE
    /opt/etc/py/getlist.py gfwlist_dec.txt $FORWARDDNS >>$GFWLISTFILE
    [ -e $TMPLISTFILE ] && rm $TMPLISTFILE
    rm gfwlist_dec.txt
    echo 'done.'
}

whitelist_py() {
    local TMPLISTFILE="/tmp/whitelist.txt"
    local whitelistFILE="/opt/etc/dnsmasq/custom/whitelist.cfg"
    wget -q --no-check-certificate -O $TMPLISTFILE https://github.com/n0wa11/gfw_whitelist/raw/master/whitelist.pac
    local TMPLISTFILESIZE=$(stat -c '%s' $TMPLISTFILE)
    if [ $TMPLISTFILESIZE -lt 1000 ] ; then
        echo "WhiteList not retrieved, exiting."
        SHR=0
        exit
    fi
    echo 'Converting whitelist to domain list...'
    echo -n "" > $whitelistFILE
    /opt/etc/py/whitelist.py $TMPLISTFILE $FORWARDDNSISP >> $whitelistFILE
    [ -e $TMPLISTFILE ] && rm $TMPLISTFILE
    echo 'done.'
}

gfwlist() {
    cd /tmp
    wget -q --no-check-certificate https://autoproxy-gfwlist.googlecode.com/svn/trunk/gfwlist.txt -O gfwlist.txt
    base64 -d gfwlist.txt > gfwlist.dec     #Base64 decode
    sed -i '1,20d' gfwlist.dec              #remove Restricted site
    grep '^@@' gfwlist.dec > gfwlist.direct #direct connect
    grep '^||' gfwlist.dec > gfwlist.p1     #pattern 1:start with "||"
    grep '^\.' gfwlist.dec > gfwlist.p2     #pattern 2:start with "."
    grep '^|[^|]' gfwlist.dec > gfwlist.p3  #pattern 3:start with single "|"
    grep -v '^|\|^$\|^@\|^\.\|^\[\|^!' gfwlist.dec > gfwlist.p4     #pattern 4:others
    #
    echo 'converting gfwlist to domain list...'
    sed -i 's/^||//g' gfwlist.p1    #remove the leading "||"
    sed -i 's/\/.*$//g' gfwlist.p1  #remove anything after "/", including "/"
    sed -i 's/\^//g' gfwlist.p1     #remove "^" if any, due to gfwlist's flawed lines
    #sed -i '/^$/d' gfwlist.p1      #delete blank line

    sed -i 's/^\.//g' gfwlist.p2    #remove the leading "."
    sed -i 's/^google[\.\*].*//g' gfwlist.p2        #remove lines start with google. or google*
    sed -i 's/\/.*//g' gfwlist.p2   #remove anything after "/", including "/"
    #sed -i '/^$/d' gfwlist.p2      #delete blank line

    sed -i 's/|http:\/\///g' gfwlist.p3     #remove prefix
    sed -i 's/|https:\/\///g' gfwlist.p3    #remove prefix
    sed -i 's/\/.*$//g' gfwlist.p3  #remove .....
    sed -i 's/^\*\.//g' gfwlist.p3
    #sed -i 's/\*//g' gfwlist.p3

    grep '\.' gfwlist.p4 > gfwlist.tmp      #remove lines contain no domain
    mv gfwlist.tmp gfwlist.p4
    sed -i 's/\/.*$//g' gfwlist.p4  #remove....
    sed -i 's/^google.*$//g' gfwlist.p4     #remove lines start with google
    grep -v '\.wikipedia\.org.*' gfwlist.p4 > gfwlist.tmp   #remove wikipedia lines
    mv gfwlist.tmp gfwlist.p4
    #sed -i '/^$/d' gfwlist.p4      #delete blank line

    cp gfwlist.p1 domainlist.tmp
    echo '
    '>> domainlist.tmp
    cat -s gfwlist.p2 >> domainlist.tmp
    echo '
    '>> domainlist.tmp
    cat -s gfwlist.p3 >> domainlist.tmp
    echo '
    '>> domainlist.tmp
    cat -s gfwlist.p4 >> domainlist.tmp
    sort domainlist.tmp | uniq > domainlist.txt
    sed -i '/^[[:space:]]*$/d' domainlist.txt       #delete blank line
    grep '\*' domainlist.txt > domainlist.special
    grep '^|' domainlist.txt >> domainlist.special
    sed -i '/\*\|^|/d' domainlist.txt
    sed -i -e "s/.*/server=\/&\/$FORWARDDNS/" domainlist.txt
    #sed -i '1s/^.*$/#### GFWList Domain for Anti-DNS-Poisoning ####/' domainlist.txt
    rm domainlist.tmp
    rm domainlist.special
    rm gfwlist.*
    mv -f /tmp/domainlist.txt /opt/etc/dnsmasq/custom/gfwlist.cfg
    echo 'done.'
}

lanternlist() {
    local TMPLISTFILE="/tmp/lanternlist.txt"
    local gfwlistFILE="/opt/etc/dnsmasq/custom/gfwlist.cfg"
    wget -q --no-check-certificate https://raw.github.com/getlantern/lantern-proxied-sites-lists/master/china/autoproxy/list.txt -O $TMPLISTFILE
    echo 'Converting lanternlist to domain list...'
    sed -i '1,10d' $TMPLISTFILE
    sed -i "s/.*/server=\/&\/$FORWARDDNS/" $TMPLISTFILE
    [ -e $gfwlistFILE ] && cat $TMPLISTFILE >> $gfwlistFILE
    sort $gfwlistFILE | uniq > $TMPLISTFILE
    mv -f $TMPLISTFILE $gfwlistFILE
    sed -i '1s/^.*$/#### GFWList Domain for Anti-DNS-Poisoning ####/' $gfwlistFILE
    echo 'done.'
}

whitelist() {
    cd /tmp
    wget -q --no-check-certificate https://github.com/n0wa11/gfw_whitelist/raw/master/whitelist.pac -O whitelist.txt
    sed -i 's/ //g' whitelist.txt        #remove all space
    grep '^"\.' whitelist.txt > whitelist.tmp #get all domain line
    echo 'converting whitelist to domain list...'
    sed -i 's/^"\.//g' whitelist.tmp   #remove the leading ""."
    sed -i 's/\/.*$//g' whitelist.tmp  #remove anything after "/", including "/"
    sed -i 's/\"//g' whitelist.tmp     #remove """ if any, due to gfwlist's flawed lines
    sed -i 's/\,//g' whitelist.tmp     #remove "," if any, due to gfwlist's flawed lines
    sort whitelist.tmp | uniq > dnswhitelist.txt
    sed -i -e "s/.*/server=\/&\/$FORWARDDNSISP/" dnswhitelist.txt
    rm whitelist.*
    mv -f /tmp/dnswhitelist.txt /opt/etc/dnsmasq/custom/whitelist.cfg
    echo 'done.'
}

pandalist() {
    local TMPLISTFILE="/tmp/panda_list.txt"
    local whitelistFILE="/opt/etc/dnsmasq/custom/whitelist.cfg"
    wget -q --no-check-certificate -O $TMPLISTFILE https://www.pandafan.org/pac/domain_list.txt
    echo 'Converting pandalist to domain list...'
    sed -i 's/\[//g' $TMPLISTFILE
    sed -i 's/\]//g' $TMPLISTFILE
    sed -i 's/, /\n/g' $TMPLISTFILE
    sed -i "s/'//g" $TMPLISTFILE
    sed -i "s/.*/server=\/&\/$FORWARDDNSISP/" $TMPLISTFILE
    [ -e $whitelistFILE ] && cat $TMPLISTFILE >> $whitelistFILE
    sort $whitelistFILE | uniq > $TMPLISTFILE
    mv -f $TMPLISTFILE $whitelistFILE
    sed -i '1 i\###Anti DNS Cache Pollution' $whitelistFILE
    [ -e $TMPLISTFILE ] && rm -f $TMPLISTFILE
    echo 'done.'
}

lazysmarthosts
tunlr
gfwlist
lanternlist
whitelist
pandalist

if [ $SHR -eq 1 ]
    then
        service dnsmasq restart
        logger "DNSMASQ restart success, new Configs and Hosts working now..."
    else
        logger "SmartHosts update failed, old Configs and Hosts working now..."
fi