Retrieves MX and A records for the 'Alexa Top 1 Million' list and prints them as nicely formatted JSON objects to stdout.
#!/usr/bin/env bash | |
# | |
# Retrieves MX and A records for the 'Alexa Top 1 Million' list | |
# and prints them as nicely formatted JSON objects to stdout. | |
# | |
# Authors: Aaron Zauner <azet@azet.org> | |
# License: CC0 1.0 (https://creativecommons.org/publicdomain/zero/1.0) | |
# | |
set -e | |
top1m_s3l="https://s3.amazonaws.com/alexa-static/top-1m.csv.zip" | |
top1m_zip=${top1m_s3l##*/} | |
top1m_csv=${top1m_zip%.*} | |
function get_mx() { | |
# build JSON object from input hostname mapping | |
# MX records to the corresponding A records of | |
# a given host. including messy in-line formatting. | |
local mx_records=($(dig +short +nosearch +keepopen \ | |
+time=2 mx "${1}" | sed 's/.*\ //')) | |
printf '{\n\t"%s": {\n\t\t"mx_records": [\n' "${1}" | |
for mx in "${mx_records[@]}"; do | |
local ip=($(getent ahostsv4 ${mx})) | |
printf '\t\t\t"%s": \t"%s",\n' "${mx}" "${ip}" | |
done | |
printf "\t\t]\n\t}\n}\n" | |
} | |
# main | |
[[ -e ${top1m_csv} ]] || { | |
wget "${top1m_s3l}" &> /dev/null | |
unzip "${top1m_zip}" &> /dev/null | |
} | |
export -f get_mx | |
export PARALLEL_SHELL=bash | |
parallel --colsep ',' --env get_mx get_mx "{2}" :::: ${top1m_csv} 2> /dev/null |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment