Blacklist from Steven Black and others convert to yml to be used with logstash translate field must contain URI / URL

convert-to-yaml-logstash.sh

#!/bin/sh
# origin https://gist.github.com/erlepereira/c11f4f7a3f60cd2071e79018e895fc8a
# logstash yaml "www.google.com": "known search engine"

# Choose from here https://github.com/StevenBlack/hosts
#HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts

##### https://raw.githubusercontent.com/StevenBlack/hosts/master/data/malwaredomainlist.com/hosts
### first must using > and others using >> for pipe
##### first file
### Master file is this, dont not move it

HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "0.0.0.0"  { print $2": adware_malware"}' $TMP_LOCATION/hosts > /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts

######

HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/data/malwaredomainlist.com/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "127.0.0.1"  { print $2": malware_domainlist"}' $TMP_LOCATION/hosts >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts

######

##0.0.0.0##HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts
HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/gambling/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "127.0.0.1"  { print $2": gambling"}' $TMP_LOCATION/hosts >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts

######

##0.0.0.0##HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/fakenews/hosts
HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/fakenews/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "0.0.0.0"  { print $2": fakenews"}' $TMP_LOCATION/hosts >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts
######

HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/social/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "127.0.0.1"  { print $2": social"}' $TMP_LOCATION/hosts >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts

######

HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/porn/clefspeare13/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "0.0.0.0"  { print $2": porn"}' $TMP_LOCATION/hosts >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts

######

HOSTS_RAW=https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/porn/sinfonietta/hosts
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 == "127.0.0.1"  { print $2": porn"}' $TMP_LOCATION/hosts >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/hosts

######

HOSTS_RAW=https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 { print $1": phishing"}' $TMP_LOCATION/phishing-domains-ACTIVE.txt >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/phishing-domains-ACTIVE.txt

######

HOSTS_RAW=https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt
TMP_LOCATION=/tmp

wget $HOSTS_RAW -P $TMP_LOCATION

awk '$1 { print $1": spam"}' $TMP_LOCATION/spammers.txt >> /etc/logstash/malware.yml

#clean up!
rm $TMP_LOCATION/spammers.txt

######

exit

# === logstash example in filter {}

#  if [query2] {
#      translate {
#        refresh_interval => 86400
#        field => "query2"
#        destination => "domain_rep"
#        dictionary_path => "/etc/logstash/malware.yml"
#      }
#  }