Retrieves MX and A records for the 'Alexa Top 1 Million' list and prints them as nicely formatted JSON objects to stdout.

get_alexa_1m_mx_rrs

#!/usr/bin/env bash
#
# Retrieves MX and A records for the 'Alexa Top 1 Million' list
# and prints them as nicely formatted JSON objects to stdout.
#
# Authors: Aaron Zauner <azet@azet.org>
# License: CC0 1.0 (https://creativecommons.org/publicdomain/zero/1.0)
#
set -e

top1m_s3l="https://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
top1m_zip=${top1m_s3l##*/}
top1m_csv=${top1m_zip%.*}

function get_mx() {
  # build JSON object from input hostname mapping
  # MX records to the corresponding A records of
  # a given host. including messy in-line formatting.
  local mx_records=($(dig +short +nosearch +keepopen \
                      +time=2 mx "${1}" | sed 's/.*\ //'))
  printf '{\n\t"%s": {\n\t\t"mx_records": [\n' "${1}"
  for mx in "${mx_records[@]}"; do
    local ip=($(getent ahostsv4 ${mx}))
    printf '\t\t\t"%s":   \t"%s",\n' "${mx}" "${ip}"
  done
  printf "\t\t]\n\t}\n}\n"
}

# main
[[ -e ${top1m_csv} ]] || {
  wget  "${top1m_s3l}" &> /dev/null
  unzip "${top1m_zip}" &> /dev/null
}

export -f get_mx
export PARALLEL_SHELL=bash
parallel --colsep ',' --env get_mx get_mx "{2}" :::: ${top1m_csv} 2> /dev/null