hosts

hosts.sh

#!/bin/bash

### Mallard's Hosts Script
###.... Create hosts.conf
###........ set LISTS to a space seperated lists of lists to download from
###.... Optionally, create custom.d
###........ Place any custom rules in files in custom.d
###........ They are interpreted as raw text
###.... Run the script as root

## Configuration
if [ ! -f "hosts.conf" ]; then
	>&2 echo "Could not find configuration file. Please create one before continuing."
	exit 1
else
	source hosts.conf
fi

if [ -z ${LISTS+x} ]; then
	>&2 echo "Malformed configuration. Please set the LISTS variable."
	exit 1
fi

if [ -z ${ALLOWED+x} ]; then
	ALLOWED=""
fi

## Pretty
echo " == LOG"

## Exit when a command fails
set -e

## Make sure that we have wget
if ! hash wget 2>/dev/null; then
	>&2 echo "Please install wget."
	exit 1
fi

## Make sure that we have permission to write to /etc/hosts
if [ "$(whoami)" != "root" ]; then
	>&2 echo "Please run this script as root."
	exit 1
fi

## Create temporary files
CUSTOM=$(mktemp)
DOWNLOAD=$(mktemp)
CLEANDOWNLOAD=$(mktemp)

## Create header
echo "### Mallard's Hosts Script" >> $CUSTOM
echo "### Generated at $(date)" >> $CUSTOM

## Create localhost rule
echo "# BEGIN localhost.rule" >> $CUSTOM
echo -e "127.0.0.1\t\tlocalhost localhost.localdomain $(hostname)" >> $CUSTOM
echo -e "# END    localhost.rule\n" >> $CUSTOM

## Apply custom rules
if [ -d "custom.d" ]; then
	for i in custom.d/*; do
		echo "Merging custom ruleset '${i}'..."
		echo "# BEGIN $i" >> $CUSTOM
		cat $i >> $CUSTOM
		echo -e "# END   $i\n" >> $CUSTOM
	done
fi

echo "# BEGIN downloads.rule" >> $CUSTOM

## Download all of the blocklists into a temporary file
for i in $LISTS; do
	echo "Downloading list '${i}'..."
	wget -q -O - $i >> $DOWNLOAD
done

## Cleanup downloads
# first expression removes comments
# second fixes line endings
# third removes lines beginning with 127.0.0.1
# fourth removes lines ending with localhost
# fifth replaces 0.0.0.0 with 127.0.0.1
# then pipe it into sort to remove duplicates
sed -e '/^#/ d' \
	-e 's/
$//' \
	-e '/^127.0.0.1/!d' \
	-e '/localhost/d' \
	-e 's/0.0.0.0/127.0.0.1/' $DOWNLOAD | sort -u > $CLEANDOWNLOAD

## Remove user specified
echo "Processing user specified blacklist..."
for i in $ALLOWED; do
	SEDRULE="/${i}/d"
	echo "A $SEDRULE"
	sed -i "${SEDRULE}" $CLEANDOWNLOAD
done

## Concatenate files
echo "Creating final list..."
cat $CUSTOM $CLEANDOWNLOAD > /etc/hosts

## Grab statistics
echo "Calculating statistics..."
NON_UNIQUE=$(wc -l < $DOWNLOAD)
UNIQUE=$(wc -l < $CLEANDOWNLOAD)

## Cleanup temporary files
echo "Cleaning up temporary files..."
rm $CUSTOM
rm $DOWNLOAD
rm $CLEANDOWNLOAD

## Pretty
echo " == STATISTICS"

## Print statistics
echo "Downloaded $NON_UNIQUE        rules."
echo "Downloaded $UNIQUE unique rules."