Intel parser

#!/usr/local/bin/bash

lists="\
DOMAIN,MalwareDomains,http://mirror1.malwaredomains.com/files/justdomains \
DOMAIN,ZeusTracker,https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist \
DOMAIN,SpyEye,https://spyeyetracker.abuse.ch/blocklist.php?download=domainblocklist
"
workdir="$PWD"
output="intel_domains.dat"
temp="tmp.txt"

function cleanup {
  if [ -e "$workdir/$temp" ]; then
    rm -f "$workdir/$temp";
  fi;
  if [ -e "$workdir/tmp"]; then
    rm -f "$workdir/tmp";
  fi;
}

cleanup

for f in $lists; do
  typ=`echo $f | awk -F "," '{print $1}'`
  src=`echo $f | awk -F "," '{print $2}'`
  url=`echo $f | awk -F "," '{print $3}'`

  wget -O tmp --no-check-certificate $url 
  cat $workdir/tmp | grep -v ^# | grep -v "^$" | sed 's/  / /g' | tr '[A-Z]' '[a-z]' | \
  awk '{print $1"\t""Intel::"typ"\t"src"\t"url"\t""F""\t""-"}' typ=$typ src=$src url=$url >> $workdir/$temp  

  rm -f $workdir/tmp
done;

# Add file header
echo "#fields   indicator       indicator_type  meta.source     meta.url        meta.do_notice  meta.if_in" > $workdir/$output

# Remove duplicates and normalize text
cat $workdir/$temp | sort -u -t$'\t' -k1,1 >> $workdir/$output

rm -f $workdir/$temp

cleanup