Bash script, using dig & curl, for reporting DNS and a few HTTPS policy files for everything email about a domain

dns-email.sh

#!/usr/bin/env bash
#
# Copyright 2020,2021,2022 Pennock Tech, LLC
# No warranty, this is a proof-of-concept not a final product.
# MIT-style license.

set -euo pipefail

# This might need to switch to another language for concurrency and handling
# the queries which are rarer, but this is a decent start as a proof-of-concept.
#
# I explicitly don't handle dig being old here.

# If the script needs more than this, then switch to using _lib.sh
progname="$(basename "$0" .sh)"
declare -i VERBOSE=0
warn() { printf >&2 '%s: %s\n' "$progname" "$*"; }
die() { warn "$@"; exit 1; }

# Needs to be POSIX ERE; RFC 5321 'sub-domain'; does not include underscore
readonly RE_VALID_DNSLABEL='[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
# DKIM selectors can be hierarchical
readonly RE_VALID_DKIM_SELECTOR="(${RE_VALID_DNSLABEL}(\.${RE_VALID_DNSLABEL})*)"
readonly WHITESPACE=$' \t'

# Inhibit curl with -C
readonly CURL_USER_AGENT='dns-email/0.1 (Phil Pennock, @philpennock GitHub)'

# If no DNSSEC confirmed for a domain, we will check this domain's DNSSEC status
# to know if it's the resolver or the domain.
readonly DNSSEC_KNOWN_SIGNED_DOMAIN='ripe.net'

declare -a DENY_DNSXL_SERVERS ALLOW_DNSXL_SERVERS ALLOW_DNSXL_DOMAIN_SERVERS
DENY_DNSXL_SERVERS=(
  zen.spamhaus.org
  dnsbl.dronebl.org
  psbl.surriel.com
  relays.nether.net
  bl.spamcop.net
  auth.spamrats.com
)
ALLOW_DNSXL_SERVERS=(
  list.dnswl.org
  swl.spamhaus.org
)
ALLOW_DNSXL_DOMAIN_SERVERS=(
  dwl.dnswl.org
)

declare -a PKIX_CERT_TA_FILES
PKIX_CERT_TA_FILES=(
  # These should contain comments making it easy to go from TLSA records
  # back to cert names.  We'll check them for the comments and discard any
  # which are missing, so it _should_ be okay to use system certs here too
  "$HOME/etc/services/x509/cert.pem"
  "/etc/ssl/cert.pem"
  "/usr/local/etc/ssl/cert.pem"
  "/etc/ssl/certs/ca-certificates.crt"
  "/etc/pki/tls/certs/ca-bundle.crt"
  "/etc/ssl/ca-bundle.pem"
  "/etc/pki/tls/cacert.pem"
  "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
)

declare -a dns_resolvers=()

usage() {
  local ev="${1:-1}"
  [[ $ev -eq 0 ]] || exec >&2
  cat <<EOUSAGE
Usage: $progname [-CLMNTcmtvx] [-A <L>] [-D <L>] [-W <L>] [-R <R>] [-i <IP>] [-s <SEL>] domain [domain ...]
 -v       be more verbose (can be repeated)
 -C       disable curl (for MTA-STS, etc)
 -M       disable assumption that MX hostnames send email
 -m       fetch mta-sts even without DNS trigger
 -N       number some resources
 -s SEL   look for a DKIM selector <SEL> in the domain
 -x       enable DNSxL checks for mail-server IPs
 -i       add IP to list of IPs to check
 -D DDL   add <DDL> to list of Deny DNSxL servers (- to blank)
 -A DAL   add <DAL> to list of Allow DNSxL servers (- to blank)
 -W DDAL  add <DDAL> to list of Domain Allow DNSxL servers (- to blank)
 -L       use local DNS resolver instead of an upstream or -R
 -t       use DNS/TCP (53/tcp) for the DNS resolver
 -T       use DNS-over-TLS (853/tcp) for the DNS resolver
 -R RES   use <RES> as DNS resolver (unless -L used)
 -c       enable color output [also triggered by \$CLICOLOR/CLICOLOR_FORCE]

Deny DNSxL servers: ${DENY_DNSXL_SERVERS[*]}
Allow DNSxL servers: ${ALLOW_DNSXL_SERVERS[*]}
Domain Allow DNSxL servers: ${ALLOW_DNSXL_DOMAIN_SERVERS[*]}
EOUSAGE
  find_diglike_command
  exit "$ev"
}

main() {
  local -i parse_options_caller_shift
  parse_options "$@"
  shift "$parse_options_caller_shift"
  if [[ $# -eq 0 ]]; then
    usage 1
  fi

  find_diglike_command
  derive_dns_resolvers

  local -A TODO_REPORT
  init_colors

  local first_output=true
  local domain
  for domain; do
    $first_output || echo
    query_domain "$domain"
    first_output=false
  done

  # These are chained so that I can keep the to-do marker near the place where
  # it belongs, but still report them to the user at the end, and only once no
  # matter how many queries they make.
  if [[ ${#TODO_REPORT[@]} -gt 0 ]]; then
    if (( VERBOSE )); then
      echo
      printf '# TODO: %s\n' "${TODO_REPORT[@]}"
    fi
  fi

  exit 0
}

# SIDE-EFFECT: sets $opt_* and $parse_options_caller_shift
# SIDE-EFFECT: updates $VERBOSE
parse_options() {
  opt_assume_mx_send=true
  opt_enable_curl=true
  opt_enumerate=false
  opt_additional_ips=()
  opt_enable_dnsxl=false
  opt_mta_sts_always=false
  opt_res_local=false
  opt_res_tcp=false
  opt_res_tls=false
  opt_resolvers=()   # we only use one at present, but permit multiple
  opt_dkim_selectors=()
  opt_enable_color=false
  if [[ -n "${CLICOLOR_FORCE:-}" ]] && [[ "$CLICOLOR_FORCE" != "0" ]]; then
    opt_enable_color=true
  elif [[ -n "${CLICOLOR:-}" ]] && [[ "$CLICOLOR" != "0" ]] && [[ -t 1 ]]; then
    opt_enable_color=true
  fi
  if [[ -n "${NO_COLOR:-${NOCOLOR:-}}" ]]; then
    opt_enable_color=false
  fi
  local arg OPTIND
  while getopts ':hci:ms:tvxA:D:CLMNR:TW:' arg; do
    case "$arg" in
      h) usage 0 ;;
      c) opt_enable_color=true ;;
      i) opt_additional_ips+=("$OPTARG") ;;
      m) opt_mta_sts_always=true ;;
      s)
        if [[ "$OPTARG" =~ ^${RE_VALID_DKIM_SELECTOR}$ ]]; then
          opt_dkim_selectors+=("$OPTARG")
        else
          die "bad -s DKIM selector '${OPTARG}'"
        fi
        ;;
      t) opt_res_tcp=true ;;
      v) VERBOSE+=1 ;;
      x) opt_enable_dnsxl=true ;;
      A) if [[ "$OPTARG" == "-" ]]; then ALLOW_DNSXL_SERVERS=(); else ALLOW_DNSXL_SERVERS+=("$OPTARG"); fi ;;
      D) if [[ "$OPTARG" == "-" ]]; then DENY_DNSXL_SERVERS=(); else DENY_DNSXL_SERVERS+=("$OPTARG"); fi ;;
      C) opt_enable_curl=false ;;
      L) opt_res_local=true ;;
      M) opt_assume_mx_send=false ;;
      N) opt_enumerate=true ;;
      R) if [[ "$OPTARG" == "-" ]]; then opt_resolvers=(); else opt_resolvers+=("$OPTARG"); fi ;;
      T) opt_res_tls=true ;;
      W) if [[ "$OPTARG" == "-" ]]; then ALLOW_DNSXL_DOMAIN_SERVERS=(); else ALLOW_DNSXL_DOMAIN_SERVERS+=("$OPTARG"); fi ;;
         # not happy at letter choice but nothing saner still left
      :) die "missing required option for -$OPTARG; see -h for help" ;;
      \?) die "unknown option -$OPTARG; see -h for help" ;;
      *) die "unhandled option -$arg; CODE BUG" ;;
    esac
  done
  parse_options_caller_shift=$((OPTIND - 1))
  shift $parse_options_caller_shift
  # any extra keyword options processing here, be sure to adjust parse_options_caller_shift
  if $opt_res_tcp && $opt_res_tls; then
    die "-t and -T are incompatible"
  fi
}

have_command() { command -v "$1" >/dev/null 2>&1 ; }

# SIDE-EFFECT: sets: $use_kdig $use_dig
find_diglike_command() {
  use_kdig=0
  use_dig=0
  if have_command kdig; then
    use_kdig=1
  elif have_command dig; then
    use_dig=1
  else
    die "fatal problem: missing a kdig or dig command"
  fi
}

# SIDE-EFFECT: sets $RRDATA[@] and $RRDATA_SAW_ALIAS
# Will skip alias type records
set_rrdata() {
  local data="${1?}"
  # bash, -g means global; zsh, means non-local.
  # We're sticking to bash, so we can't have a maskable array without just assigning it
  RRDATA=()
  RRDATA_SAW_ALIAS=false
  if [[ "$data" == "" ]]; then
    return
  fi
  local rrname ttl class rrtype rrdata
  while read -r rrname ttl class rrtype rrdata; do
    case "${rrtype^^}" in
      CNAME) RRDATA_SAW_ALIAS=true ;;
      DNAME) RRDATA_SAW_ALIAS=true ;;
      *) RRDATA+=("$rrdata") ;;
    esac
  done <<<"$data"
}

# SIDE-EFFECT: sets $MX_HOSTS[@]
# SIDE-EFFECT: indirectly causes $RRDATA[@] to be set
set_mx_hosts() {
  local RRDATA
  set_rrdata "${1:?}"
  MX_HOSTS=()
  local mx
  local pattern='^[[:digit:]]+[[:space:]]+([^[:space:]]+)$'
  for mx in "${RRDATA[@]}"; do
    if [[ $mx == "0 ." ]]; then
      printf '#  NULL MX, no email accepted here\n'
      continue
    fi
    [[ $mx =~ $pattern ]] || continue
    MX_HOSTS+=("${BASH_REMATCH[1]}")
    derive_implicit_dkim_mx "${BASH_REMATCH[1]}"
  done
}

# SIDE-EFFECT: adds to $TODO_REPORT[@]
query_domain() {
  local domain="${1:?}"
  local -l lower_domain="$domain"
  local RRDATA MX_HOSTS MX_IPS IMPLICIT_DKIM_SELECTORS
  local domain_has_apex_address
  local mx_lines
  local d s t saw_tlsa ip rev_ip

  printf '# Domain: %s\n' "$domain"
  if domain_is_signed "$domain" ; then
    printf '#  is DNSSEC signed\n'
  elif resolver_not_verifying; then
    printf '#  (our resolver does not verify DNSSEC)\n'
  else
    printf '#  is *NOT* DNSSEC signed\n'
  fi
  local mx_lines
  mx_lines="$(dig_one -t mx "$domain")"
  if [[ -z "$mx_lines" ]]; then
    printf '#   No MX records found??\n'
  else
    set_mx_hosts "$mx_lines"
    printf '# MX:\n%s\n' "$mx_lines"

    # This is a heuristic: we really care about the reverse DNS of the hosts
    # which _send_ email _outbound_ from a domain, but we don't have a means of
    # identifying those.  For sufficiently small mail domains, the MX hosts will
    # also be the sending hosts, so check the MX hosts for reverse DNS.
    #
    # This will give false positive complaints for large providers.
    #
    # We don't care about reverse DNS of the MX host which we connect outbound to,
    # folks don't do spam checks on who they send to, only whom they receive from.
    if $opt_assume_mx_send; then
      if inhibit_assume_mx_send_by_mx "${MX_HOSTS[@]}"; then
        printf '# Matched MX "%s", MX are not senders\n' "$INHIBIT_MATCH"
      else
        check_mx_hosts "$domain"
        printf '# MX IP addresses: %s\n' "${MX_IPS[*]}"
      fi
    fi

  fi # missing MX guard

  printf '\n'

  t="$(dig_one -t a "$domain"; dig_one -t aaaa "$domain")"
  if [[ -n "$t" ]]; then
    domain_has_apex_address=true
  else
    domain_has_apex_address=false
  fi

  printf '# SPF:\n'
  chase_spf "$domain"
  # do we want to check if the MX host is in the SPF records?  Would require
  # iterating (with depth/query limits) and finding CIDR blocks and checking
  # various other SPF entry types.
  # We should also have a "get MX_IPS" approach which we can call if not
  # checking MX hosts for forward/reverse DNS.

  printf '# DMARC/historical/other policies:\n'
  dig_one_txt_else _dmarc "$domain" 'v=DMARC1' DMARC
  t="$(dig_one -t txt "_dmarc.${domain}")"  # this is the exact same check just done, we lose the shell-layer cache to avoid re-architect, but it should at least be readily available in DNS caches.
  # RFC 7489 §7.1 requires that a target domain which isn't ours have authorization records.
  # Without implementing PSL checks, I'm going to target the common case of unrelated domains, rather than sub-domains of a public domain.
  # We can fix this logic if it triggers too many false positives because of sub-domain usage within one public domain.
  local dmarc_alt_checked=''
  if [[ "$t" =~ rua=mailto:([^${WHITESPACE};]+)(;.*)?$ ]]; then  # FIXME: the value can be a comma-separated list, need to handle that.
    d="${BASH_REMATCH[1]}"
    d="${d##*@}"
    if [[ "$d" != "$domain" ]]; then
      dig_one_txt_else "$domain._report._dmarc" "$d" 'v=DMARC1' 'DMARC authorization'
      dmarc_alt_checked="$d"
    fi
  fi
  if [[ "$t" =~ ruf=mailto:([^${WHITESPACE};]+)(;.*)?$ ]]; then  # FIXME: the value can be a comma-separated list, need to handle that.
    d="${BASH_REMATCH[1]}"
    d="${d##*@}"
    if [[ "$d" != "$domain" ]] && [[ "$d" != "$dmarc_alt_checked" ]]; then
      dig_one_txt_else "$domain._report._dmarc" "$d" 'v=DMARC1' 'DMARC authorization'
      dmarc_alt_checked="$d"
    fi
  fi
  dig_one_txt_else _adsp._domainkey "$domain" '' ADSP
  dig_one_txt_else default._bimi "$domain" 'v=BIMI1' BIMI

  # Common pattern
  local postmaster
  postmaster="$(dig_one -t a "postmaster.$domain"; dig_one -t aaaa "postmaster.$domain"; true)"
  if [[ -n "$postmaster" ]]; then
    printf '# postmaster exists in DNS, try <https://postmaster.%s/>\n' "$domain"
    printf '%s\n' "$postmaster"
  else
    printf '# postmaster hostname not found in DNS\n'
  fi

  # draft-ietf-marid-csv-csa-01 from 2004; went nowhere as a standard AFAIK but some use it
  printf '# CSA:\n'
  dig_one -t srv "_client._smtp.$domain"

  local -a all_selectors dedup_selectors
  local -A done_selector
  all_selectors=("${opt_dkim_selectors[@]}" "${IMPLICIT_DKIM_SELECTORS[@]}")
  if [[ "${#all_selectors[@]}" -gt 0 ]]; then
    for s in "${all_selectors[@]}"; do
      if ${done_selector[$s]:-false}; then continue; fi
      dedup_selectors+=("$s")
      done_selector["$s"]=true
    done
    printf '\n# DKIM records for selectors: %s\n' "${dedup_selectors[*]}"
    for s in "${dedup_selectors[@]}"; do
      t="$(dig_one -t txt "${s}._domainkey.${domain}" || true)"
      show_or_comment "$t" "MISSING [$s]"
    done
  fi

  # RFC 6186 for the framework; RFC 8314 adds submissions and strongly discourages non-TLS
  printf '\n# Client SRV records:\n'
  for s in submissions submission imaps imap pop3s pop3 sieve; do
    dig_one -t srv "_${s}._tcp.$domain"
  done

  # draft-koch-openpgp-webkey-service
  printf '\n# User identity:\n'
  wkd_check "$domain" "$domain_has_apex_address"

  # RFC 8659 (with extensions in 8657)
  printf '\n# TLS Policies:\n'
  t="$(dig_one -t caa "$domain")"
  show_or_comment "$t" "No CAA records found"
  # FIXME: need an AD check for DNSSEC here
  saw_tlsa=false
  for h in "${MX_HOSTS[@]}"; do
    t="$(dig_one -t tlsa "_25._tcp.$h")"
    if [[ -n "$t" ]]; then
      render_tlsa_records "$t"
      saw_tlsa=true
    fi
  done
  if ! $saw_tlsa; then
    printf '#   No TLSA records found\n'
  fi

  # RFC 8461
  t="$(dig_one -t txt "_mta-sts.$domain")"
  show_or_comment "$t" "No MTA-STS enabling TXT trigger found"
  dig_one -t a "mta-sts.$domain"
  dig_one -t aaaa "mta-sts.$domain"
  local do_mtasts_fetch=false
  if $opt_enable_curl && contains_rrtype TXT "$t" 'v=STSv1'; then
    do_mtasts_fetch=true
  elif [[ -n "$t" ]] && ! contains_rrtype TXT "$t"; then
    printf '#   MTA-STS enabling TXT trigger results missing TXT record (bad wildcard?)\n'
  elif [[ -n "$t" ]] && ! contains_rrtype TXT "$t" 'v=STSv1'; then
    printf '#   MTA-STS enabling TXT trigger missing enable prefix, ignoring\n'
  fi
  if $opt_enable_curl && $opt_mta_sts_always && ! $do_mtasts_fetch; then
    printf '#   MTA-STS fetch triggered explicitly by caller anyway\n'
    do_mtasts_fetch=true
  fi
  if "$do_mtasts_fetch"; then
    t="$(cmd_curl_trace "https://mta-sts.$domain/.well-known/mta-sts.txt" || true )"
    if [[ -z "$t" ]]; then
      printf '#   no mta-sts.txt retrieved?\n'
    else
      printf '%s\n' "$t" | sed 's/^/    mta-sts: /'
    fi
  fi

  # RFC 8460: SMTP TLS Reporting
  dig_one_txt_else _smtp._tls "$domain" 'v=TLSRPTv1' TLS-RPT

  if $opt_enable_dnsxl; then
    printf '\n'
    printf '# DNSxL checks: allow-lists (presence _good_):\n'
    for ip in "${MX_IPS[@]}" "${opt_additional_ips[@]}"; do
      rev_ip="$(reverse_ip "$ip")" || continue
      if (( VERBOSE )); then printf '#   IP=[%s] => reversed IP [%s]\n' "$ip" "$rev_ip"; fi
      for dnsxl in "${ALLOW_DNSXL_SERVERS[@]}"; do
        if (( VERBOSE )); then printf '#    DNSxL=<%s>\n' "$dnsxl"; fi
        dig_one -t a "${rev_ip}.${dnsxl}"
        dig_one -t txt "${rev_ip}.${dnsxl}"
      done
    done
    printf '# DNSxL checks: domain-allow-lists (presence _good_):\n'
    for dnsxl in "${ALLOW_DNSXL_DOMAIN_SERVERS[@]}"; do
      if (( VERBOSE )); then printf '#    DNSxL=<%s>\n' "$dnsxl"; fi
        dig_one -t a "${domain}.${dnsxl}"
        dig_one -t txt "${domain}.${dnsxl}"
    done
    printf '# DNSxL checks: deny-lists (presence **BAD**):\n'
    for ip in "${MX_IPS[@]}" "${opt_additional_ips[@]}"; do
      rev_ip="$(reverse_ip "$ip")" || continue
      if (( VERBOSE )); then printf '#   IP=[%s] => reversed IP [%s]\n' "$ip" "$rev_ip"; fi
      for dnsxl in "${DENY_DNSXL_SERVERS[@]}"; do
        if (( VERBOSE )); then printf '#    DNSxL=<%s>\n' "$dnsxl"; fi
        dig_one -t a "${rev_ip}.${dnsxl}"
        dig_one -t txt "${rev_ip}.${dnsxl}"
      done
    done
  fi
}

show_or_comment() {
  local body comment returncode
  body="${1-}"  # empty explicitly okay
  comment="${2:?}"
  returncode="${3:-false}"
  if [[ -n "$body" ]]; then
    printf '%s\n' "$body"
    if [[ "$body" =~ '\\"' ]]; then
      printf '#\n#   WARNING: LOOKS LIKE EMBEDDED LITERAL QUOTES IN THAT!\n#\n'
    fi
    if $returncode; then return 0; fi
  else
    printf '#   %s\n' "$comment"
    if $returncode; then return 1; fi
  fi
}

show_or_comment_check_rrtype_starts() {
  local body rrtype prefix system
  body="${1-}" rrtype="${2:?}" prefix="${3:-}" system="${4:?}"
  if show_or_comment "$body" "No $system $rrtype record found" true; then
    if ! contains_rrtype "$rrtype" "$body" "$prefix"; then
      printf '#   no "%s" in a %s record, so no %s\n' "$prefix" "$rrtype" "$system"
    fi
  fi
}

dig_one_txt_else() {
  local sublabel domain seekprefix label
  sublabel="${1:?}" domain="${2:?}" seekprefix="${3:-}" label="${4:?}"
  local t
  t="$(dig_one -t txt "${sublabel}.${domain}")"
  show_or_comment_check_rrtype_starts "$t" TXT "$seekprefix" "$label"
}

contains_rrtype() {
  local want_rrtype="${1:?}"
  local data="${2-}"
  local prefix="${3:-}"
  [[ -n "$data" ]] || return 1

  local rrname ttl class rrtype rrdata
  local -u needle this_rrtype
  needle="$want_rrtype"

  while read -r rrname ttl class rrtype rrdata; do
    echo >/dev/null "shellcheck silence unused variables, named for identification not because need them: $rrname $ttl $class"
    this_rrtype="$rrtype"
    [[ "$this_rrtype" == "$needle" ]] || continue
    if [[ -n "${prefix:-}" ]]; then
      case "$rrdata" in
        "$prefix"*) return 0 ;;
        \""$prefix"*) return 0 ;;
        *) continue ;;
      esac
    fi
    return 0
  done <<<"$data"

  return 1
}

# SIDE-EFFECT: sets $MX_IPS[@]
# SIDE-EFFECT: sets $MX_CONTAINS_ALIAS
check_mx_hosts () {
  local domain="${1:?}"

  local RRDATA REVERSE_ERRORS REVERSE_WARNINGS PTR_BLOCKS
  local addr_lines_a addr_lines_aaaa one_mx ADDRS

  # Deliberate leak; we use it here, but let callers see it too
  MX_CONTAINS_ALIAS=false

  printf '# MX hosts:\n'
  ADDRS=()
  REVERSE_ERRORS=()
  REVERSE_WARNINGS=()
  PTR_BLOCKS=()
  declare -A checked_mx_hosts
  for one_mx in "${MX_HOSTS[@]}"; do
    if [[ "${checked_mx_hosts[$one_mx]:+yes}" == "yes" ]]; then
      if (( VERBOSE )); then
        printf '## [%s] SKIPPING duplicate MX hostname: %s\n' "$domain" "$one_mx"
      fi
      continue
    fi
    checked_mx_hosts[$one_mx]=t
    addr_lines_a="$(dig_one -t a "$one_mx")"
    addr_lines_aaaa="$(dig_one -t aaaa "$one_mx")"
    printf '%s\n' "$addr_lines_a" "$addr_lines_aaaa"

    set_rrdata "$addr_lines_a"
    if "$RRDATA_SAW_ALIAS"; then MX_CONTAINS_ALIAS=true; fi
    ADDRS+=("${RRDATA[@]}")
    check_mxhost_reverse_dns "$one_mx" A "${RRDATA[@]}"

    set_rrdata "$addr_lines_aaaa"
    ADDRS+=("${RRDATA[@]}")
    check_mxhost_reverse_dns "$one_mx" AAAA "${RRDATA[@]}"
  done
  if $MX_CONTAINS_ALIAS; then
    printf '#\n%s# *** ERROR: MX records contained an alias rrtype ***%s\n#\n' "$COLOR_START_ERR" "$COLOR_END"
  fi
  echo
  printf '# MX Reverse DNS:\n'
  printf '%s\n' "${PTR_BLOCKS[@]}"
  if [[ ${#REVERSE_ERRORS[@]} -eq 0 ]] && [[ ${#REVERSE_WARNINGS[@]} -eq 0 ]]; then
    printf '# no problems with reverse DNS\n'
  else
    if [[ ${#REVERSE_ERRORS[@]} -gt 0 ]]; then
      printf '#\n# Reverse DNS errors:%s\n' "$COLOR_START_ERR"
      printf '#ERROR: %s\n' "${REVERSE_ERRORS[@]}"
      printf '%s' "$COLOR_END"
    fi
    if [[ ${#REVERSE_WARNINGS[@]} -gt 0 ]]; then
      printf '#\n# Reverse DNS minor issues:%s\n' "$COLOR_START_WARN"
      printf '#warning: %s\n' "${REVERSE_WARNINGS[@]}"
      printf '%s' "$COLOR_END"
    fi
  fi

  MX_IPS=("${ADDRS[@]}")
}

# SIDE-EFFECT: appends to $REVERSE_ERRORS[@]
# SIDE-EFFECT: appends to $REVERSE_WARNINGS[@]
# SIDE-EFFECT: appends to $PTR_BLOCKS[@]
check_mxhost_reverse_dns() {
  local mxhost="${1:?}"
  local rrtype="${2:?}"
  shift 2
  local RRDATA

  if [[ $# -eq 0 ]]; then
    REVERSE_ERRORS+=("[$mxhost] missing forward $rrtype DNS")
    return
  fi
  local addr ptr_lines missing_mxhost entry
  for addr; do
    ptr_lines="$(dig_one -x "$addr")"
    if [[ -z "$ptr_lines" ]]; then
      REVERSE_ERRORS+=("[$mxhost] no reverse DNS for address: $addr")
      continue
    fi
    PTR_BLOCKS+=("$ptr_lines")
    set_rrdata "$ptr_lines"
    # CNAMEs are definitely okay here, that's how non-octet reverse delegation is done
    local -a FOUND_PTRS
    FOUND_PTRS=("${RRDATA[@]}")
    missing_mxhost=true
    for entry in "${FOUND_PTRS[@]}"; do
      if [[ "$entry" == "$mxhost" ]]; then
        missing_mxhost=false
        break
      elif (( VERBOSE >= 2 )); then
        printf '### [%s] no-match: seeking %s in: %s (from %s)\n' "$mxhost" "$mxhost" "$entry" "$addr"
      fi
    done

    if $missing_mxhost; then
      if (( VERBOSE )); then
        # We only really care about hostnames which can be seen from the IP
        # address, which is what servers receiving an inbound MX connection see
        # as their starting point.  So having an MX hostname have reverse DNS
        # which matches the hostname used to connect inbound to the domain is
        # not something anyone ever truly checks.
        REVERSE_WARNINGS+=("[$mxhost] (minor non-issue) reverse DNS missing MX hostname: $addr")
      fi

      local forward_lines forward_ip forward_entry
      local missing_match_forward=true
      for forward_entry in "${FOUND_PTRS[@]}"; do
        forward_lines="$(dig_one -t "$rrtype" "$forward_entry")"
        if [[ -n "$forward_lines" ]]; then
          PTR_BLOCKS+=("$forward_lines")  # I know, not PTRs
          set_rrdata "$forward_lines"
          for forward_ip in "${RRDATA[@]}"; do
            if [[ "$forward_ip" == "$addr" ]]; then
              missing_match_forward=false
              break
            elif (( VERBOSE >= 2 )); then
              printf '### [%s] no-match: seeking %s in %s (from %s)\n' "$mxhost" "$addr" "$forward_ip" "$forward_entry"
            fi
          done
        elif (( VERBOSE )); then
          printf '### [%s] reverse DNS from %s has host %s which has no %s records\n' "$mxhost" "$addr" "$forward_entry" "$rrtype"
        fi # have forward DNS for a ptr
      done

      if $missing_match_forward; then
        REVERSE_ERRORS+=("[$mxhost] address $addr lacks matching forward DNS")
      fi
    fi

  done
}

# SIDE-EFFECT: sets $INHIBIT_MATCH
inhibit_assume_mx_send_by_mx() {
  local x
  local -l lx
  for x; do
    lx="$x"
    case "$lx" in
    aspmx.l.google.com.) INHIBIT_MATCH="$x"; return 0 ;;
    gmr-smtp-in.l.google.com.) INHIBIT_MATCH="$x"; return 0 ;;  # email forwarding servers
    in*-smtp.messagingengine.com.) INHIBIT_MATCH="$x"; return 0 ;;
    esac
  done
  return 1
}

# SIDE-EFFECT: appends to $IMPLICIT_DKIM_SELECTORS[@]
chase_spf() {
  local domain="${1:?}"
  shift
  local -i SPF_COUNT=0 SPF_DNS_QCOUNT=0 SPF_NEST_DEPTH=0
  local -A CHECKED_SPF
  local SPF_ABORT=false

  # The limit of 10 is supposed to be for any terms which cause DNS queries,
  # and it's a query limit, not a recursion limit, so these:
  #  mechanisms: include, a, mx, ptr, exists
  #  modifiers: redirect
  # all count towards the limit.
  # Here, SPF_COUNT is how many SPF records we see, SPF_DNS_QCOUNT is how many
  # DNS records are looked up.

  # Spam Assassin uses a limit of 20 because of routine violations, see
  # <https://bz.apache.org/SpamAssassin/show_bug.cgi?id=7182>
  # Find real current value in source, lib/Mail/SpamAssassin/Plugin/SPF.pm
  # sub _check_spf, around line 530, `max_dns_interactive_terms` value
  # rspamd has a total limit and a _nesting_ limit.
  # src/libserver/spf.h : SPF_MAX_NESTING=10 SPF_MAX_DNS_REQUESTS=30 (defaults, can be overruled)
  local -i SPF_QCOUNT_WARN=10
  local SPF_WARN_MSG='should be at most 10'

  SPF_DNS_QCOUNT+=1
  dig_one_spf_domain "$domain"

  TODO_REPORT['spf_chasing']="SPF: handle exp=<explain> expandos"
  TODO_REPORT['spf_quotes']="SPF: remove entirely space between quotes, better handle multi-string"
}

dig_one() {
  local cmd
  local -a flags at_res
  flags=( +nodnssec +nonsid +nomultiline +noall +answer )
  if (( use_kdig )); then
    cmd=kdig
    flags+=( +nocomments )
    if $opt_res_tls; then
      flags+=( +tls )
    fi
  fi
  if (( use_dig )); then
    cmd=dig
    flags+=( +nosearch )
    if $opt_res_tls; then
      die "can't use +tls with dig(1), sorry"
    fi
  fi
  if $opt_res_tcp; then
    flags+=( +tcp )
  fi
  if [[ "${#dns_resolvers[@]}" -ge 1 ]]; then
    at_res+=("@${dns_resolvers[0]}")
  fi
  # echo >&2 "running: $cmd ${at_res[*]} ${flags[*]} $*"
  command "$cmd" "${at_res[@]}" "${flags[@]}" "$@"
}

# SIDE-EFFECT: modifies $SPF_COUNT, $CHECKED_SPF[@], $SPF_ABORT, $IMPLICIT_DKIM_SELECTORS[@]
dig_one_spf_domain() {
  local domain="${1:?}"
  if [[ "${CHECKED_SPF[$domain]:+yes}" == "yes" ]]; then
    # allow for diamond includes, don't abort (but could usefully look for loops)
    $SPF_ABORT || printf '# Not checking twice for SPF: %s\n' "$domain"
    return
  fi
  local -i SPF_NEST_DEPTH="$SPF_NEST_DEPTH"
  SPF_NEST_DEPTH+=1
  derive_implicit_dkim_spfdomain "$domain"
  CHECKED_SPF[$domain]='pre'
  SPF_COUNT+=1
  if $SPF_ABORT; then return; fi
  if (( SPF_DNS_QCOUNT > SPF_QCOUNT_WARN )); then
    # nb: there are often A+AAAA lookups, but they count  as just 1.
    printf '%s# ERROR: too many [%d] DNS lookups for SPF, %s%s\n' "$COLOR_START_ERR" "$SPF_DNS_QCOUNT" "$SPF_WARN_MSG" "$COLOR_END"
    if (( SPF_QCOUNT_WARN == 10 )); then
      SPF_QCOUNT_WARN=20
      SPF_WARN_MSG="exceeded SpamAssassin's limit"
    elif (( SPF_QCOUNT_WARN == 20 )); then
      SPF_QCOUNT_WARN=30
      SPF_WARN_MSG="exceeded rspamd total query limit"
    elif (( SPF_QCOUNT_WARN == 30 )); then
      SPF_QCOUNT_WARN=99999
      SPF_WARN_MSG='wth?'
    fi
  fi
  if (( SPF_NEST_DEPTH == 10 )); then
    # we can hit this multiple times on different branches, but should only report once per branch.
    printf '%s# ERROR: exceeded rspamd nesting depth limit [%d]%s\n' "$COLOR_START_ERR" "$SPF_NEST_DEPTH" "$COLOR_END"
  fi
  if (( SPF_COUNT >= 100 )); then
    SPF_ABORT=true
    printf '# ... and that makes 100, too many to keep reporting for diagnosis\n'
    return
  fi

  local out fout part lpart last
  local -l lpart last
  while read -r out; do
    [[ -n "$out" ]] || continue
    fout="${out//\"/}"
    # shellcheck disable=SC2086
    set -- $fout
    shift 4  # rrname, ttl, class, rrtype
    case "$1" in
      v=spf1*)
        if $opt_enumerate; then printf '%3d/%3d  ' "$SPF_COUNT" "$SPF_DNS_QCOUNT"; fi
        printf '%s\n' "$out"
        ;;
      *)
        if (( VERBOSE >= 3 )); then printf '# excluded: %s\n' "$out"; fi
        continue
        ;;
    esac
    for part; do
      lpart="$part"  # auto-lowered
      last="$part"
      case "$lpart" in
        include:*) SPF_DNS_QCOUNT+=1; dig_one_spf_domain "${lpart#include:}" ;;
        a | a:* | mx | mx:* | ptr | ptr:*) SPF_DNS_QCOUNT+=1 ;;
        exists:* ) SPF_DNS_QCOUNT+=1 ;;
      esac
    done
    case "$last" in
      redirect=*) SPF_DNS_QCOUNT+=1; dig_one_spf_domain "${last#redirect=}" ;;
    esac
  done <<<"$(dig_one -t txt "$domain")"

  CHECKED_SPF[$domain]='done'
}

# SIDE-EFFECT: appends to $IMPLICIT_DKIM_SELECTORS[@]
derive_implicit_dkim_spfdomain() {
  # This one is less useful and mostly a relic of some past practices which are dying out.
  # SPF is for the SMTP Envelope Sender, which needs to point back to that mail sending service for bounces, so the only SPF records which could would be them.
  # *If* you pattern-match in your MTA and route some bounce messages back to the sending service then this still helps, but that's fairly rare.
  # For a while, folks were including records for all outsourced senders in their own SPF record.  It's normally pointless though.
  # Eg, mailchimp have stopped providing an interface to walk people through SPF setup.  They're DKIM-only.
  # So:
  #   1. For the MX case, that's covered in another function
  #   2. For some rare cases where folks align the sender and forward bounces, this does help
  #   3. This might also help with vanity MX records?
  #   4. For common case, this is unlikely to do too much
  # Still, it's all just hints, so we might as well take the hints.
  local domain="${1:?}"
  case "$domain" in
    _spf.google.com) IMPLICIT_DKIM_SELECTORS+=(google) ;;
    mail.zendesk.com) IMPLICIT_DKIM_SELECTORS+=(zendesk1 zendesk2) ;;
    servers.mcsv.net) IMPLICIT_DKIM_SELECTORS+=(k1 k2 k3) ;;  # mailchimp
    mktomail.com) IMPLICIT_DKIM_SELECTORS+=(m1) ;; # marketo.com
    spf.protection.outlook.com | spf.protection.outlook.de ) IMPLICIT_DKIM_SELECTORS+=(selector1 selector2) ;;
    spf.messagingengine.com) IMPLICIT_DKIM_SELECTORS+=(fm1 fm2 fm3) ;;  # fastmail
    sendgrid.net) IMPLICIT_DKIM_SELECTORS+=(s1 s2) ;; # sendgrid defaults to s1/s2 but allows up to 3 chars of custom selector
    spf.mandrillapp.com) IMPLICIT_DKIM_SELECTORS+=(mandrill) ;;
    spf.zoho.com | zoho.com) IMPLICIT_DKIM_SELECTORS+=(zoho) ;; # not forced, it's a walkthrough, but they suggest this and I suspect most go along with that.
    *cust.gandi.net) IMPLICIT_DKIM_SELECTORS+=(gm1 gm2 gm3) ;;

    # ADDING: SHOULD NOT HAVE TRAILING DOT ON MATCHER

    # amazonses.com) ;;        # long random selectors, nothing implicit
    # mailgun.org) ;;          # user-selectable
    # _spf.salesforce.com) ;;  # user-selectable
  esac
}

# SIDE-EFFECT: appends to $IMPLICIT_DKIM_SELECTORS[@]
derive_implicit_dkim_mx() {
  local mxhost="${1:?}"
  case "$mxhost" in
    aspmx.l.google.com.) IMPLICIT_DKIM_SELECTORS+=(google) ;;
    in[12]-smtp.messagingengine.com.) IMPLICIT_DKIM_SELECTORS+=(fm1 fm2 fm3) ;;  # fastmail
    *.mail.gandi.net.) IMPLICIT_DKIM_SELECTORS+=(gm1 gm2 gm3) ;;

    # ADDING: REQUIRES TRAILING DOT ON MATCHER
  esac
}

wkd_check() {
  local domain="${1:?}"
  local has_apex_address="${2:?}"
  local t u

  t="$(dig_one -t a "openpgpkey.$domain"; dig_one -t aaaa "openpgpkey.$domain")"
  printf '%s\n' "${t:-# No openpgpkey.$domain found}"

  $opt_enable_curl || return 0

  if [[ -n "$t" ]]; then
    u="https://openpgpkey.$domain/.well-known/openpgpkey/$lower_domain/policy"
  elif $has_apex_address; then
    u="https://$domain/.well-known/openpgpkey/policy"
  else
    return 0
  fi

  # Ideally we'd have curl print each URL tried, as it tries them, along a location redirect chain
  if t="$(cmd_curl --max-redirs 5 -L --max-time 5 "$u")"; then
    printf '#   <%s>\n' "$u"
    if [[ -n "$t" ]]; then
      printf '%s\n' "$t" | sed -e 's/^/    wkd-policy: /' -e $'16c \\\n        [TRUNCATED]' -e '17{x;q}'
    else
      printf '#   wkd policy exists and is empty (which is fine and reasonable)\n'
    fi
  else
    printf '# no WKD policy found (tried <%s>) [%s]\n' "$u" "$?"
  fi
}

render_tlsa_records() {
  local raw_records="${1:?}"
  local -r tlsa_pat='\bTLSA[[:space:]]+([[:digit:]]+)[[:space:]]+([[:digit:]]+)[[:space:]]+([[:digit:]]+)[[:space:]]+(.+)$'
  local tlsa_usage tlsa_selector tlsa_matching tlsa_cadata
  local tafile known name
  local -a have_tafiles=()

  # Check to see which of our augmented trust-anchor files exist
  # If many do (unlikely! typically 1, or 2 if my own is present)
  # then we should expect this file to be in buffer cache most of the time, so
  # a grep should be fast.  Thus I'm not bothering to optimize this out of the
  # loop, to ensure we only filter once if invoked against multiple domains.
  for tafile in "${PKIX_CERT_TA_FILES[@]}"; do
    [[ -f "$tafile" ]] || continue
    grep -q '^# TLSA' -- "$tafile" || continue
    have_tafiles+=("$tafile")
  done

  if [[ "${#have_tafiles[@]}" -eq 0 ]]; then
    # None have our TLSA comments letting us index-scan by what we found in DNS.
    printf '%s\n' "$raw_records"
    return
  fi

  printf '%s\n' "$raw_records" | while read -r line; do
    known='' name=''
    if [[ "$line" =~ $tlsa_pat ]]; then
      tlsa_usage="${BASH_REMATCH[1]}"
      tlsa_selector="${BASH_REMATCH[2]}"
      tlsa_matching="${BASH_REMATCH[3]}"
      tlsa_cadata="${BASH_REMATCH[4]}"
      tlsa_cadata="${tlsa_cadata// /}"

      echo >/dev/null "shellcheck silence unused variables, named for identification not because need them: $tlsa_selector $tlsa_matching"

      case "$tlsa_usage" in
        0) ;; # PKIX-TA
        2) ;; # DANE-TA
        *) printf '%s\n' "$line"; continue ;;
      esac
      # Anything here is theoretically a trust-anchor.

      for tafile in "${have_tafiles[@]}"; do
        [[ -f "$tafile" ]] || continue
        known="$(grep -FiC5 -m1 "$tlsa_cadata" -- "$tafile")" || continue
        break
      done
      if [[ -z "$known" ]]; then
        printf '%s\n' "$line"
        continue
      fi
      name="$(sed -En 's/# *Name *: *(.+)/\1/p' <<<"$known")"
      if [[ -z "$name" ]]; then
        printf '%s\n' "$line"
      else
        printf '%s ; %s\n' "$line" "$name"
      fi

    else
      printf '%s\n' "$line"
    fi
  done
}

reverse_ip() {
  local ip="$1"
  local q1 q2 q3 q4
  shift
  if [[ $ip =~ ^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then
    q1=$((0 + BASH_REMATCH[1]))
    q2=$((0 + BASH_REMATCH[2]))
    q3=$((0 + BASH_REMATCH[3]))
    q4=$((0 + BASH_REMATCH[4]))
    printf '%s.%s.%s.%s\n' "$q4" "$q3" "$q2" "$q1"
    return 0
  fi
  return 1
}

cmd_curl() {
  command curl \
    -H "User-Agent: $CURL_USER_AGENT" \
    --max-time 5 \
    -fSs "$@"
}

cmd_curl_trace() {
  local x
  for x; do true; done
  printf '#   <%s>\n' "$x"
  cmd_curl "$@"
}

# unused(?) but keeping around
dig_each() {
  local -a params
  while [[ "$1" != '--' ]]; do
    params+=("$1")
    shift
  done
  shift
  local x
  for x; do
    dig_one "${params[@]}" "$x"
  done
}

# unused(?) but keeping around
dig_one_prefixfilter() {
  local filter="${1:?}"
  shift
  local out fout
  dig_one "$@" | while read -r out; do
    fout="${out//\"/}"
    # shellcheck disable=SC2086
    set -- $fout
    case $5 in
      ${filter}*) printf '%s\n' "$out" ;;
      *) if (( VERBOSE >= 3 )); then printf '# excluded: %s\n' "$out"; fi ;;
    esac
  done
}

domain_is_signed() {
  local domain="${1:?}"
  local header_opt res flags
  if (( use_kdig )); then
    header_opt='+header'
  else
    header_opt='+comments'
  fi
  res="$(dig_one "$header_opt" +adflag -t mx "$domain")"
  # dig emits 'flags', kdig emits 'Flags'
  flags="$(sed -En 's/^;; [Ff]lags: ([^;]+);.*/\1/p' <<<"$res")"
  grep -qiE '\bad\b' <<<"$flags"
}

# SIDE-EFFECT: updates $cached_resolver_not_verifying_rv
resolver_not_verifying() {
  if [[ -n "${cached_resolver_not_verifying_rv:-}" ]]; then
    return "$cached_resolver_not_verifying_rv"
  fi
  if domain_is_signed "$DNSSEC_KNOWN_SIGNED_DOMAIN"; then
    # domain is signed, so our negate sense predicate is false, so set non-zero
    cached_resolver_not_verifying_rv=1
  else
    cached_resolver_not_verifying_rv=0
  fi
  return "$cached_resolver_not_verifying_rv"
}

derive_dns_resolvers() {
  # This is a script of mine which finds real DNS resolvers
  # instead of 127.0.0.53
  if ! $opt_res_local; then
    if [[ "${#opt_resolvers[@]}" -gt 0 ]]; then
      dns_resolvers=("${opt_resolvers[@]}")
    else
      # I know the exact text format used here, so:
      # shellcheck disable=SC2207
      dns_resolvers=($( dns-resolvers 2>/dev/null )) || true
    fi
    if (( VERBOSE )); then
      if (( "${#dns_resolvers[@]}" )); then
        printf '# [using DNS resolver: %s]\n' "${dns_resolvers[0]}"
      else
        printf '# using system DNS resolver\n'
      fi
    fi
  fi
}

# SIDE-EFFECT: sets variables starting COLOR_
init_colors() {
  if $opt_enable_color; then
    declare -gr COLOR_START_ERR=$'\033[31;1m' COLOR_START_WARN=$'\033[35;1m' COLOR_END=$'\033[m'
  else
    declare -gr COLOR_START_ERR='' COLOR_START_WARN='' COLOR_END=''
  fi
}

# Don't run main if sourced; easier to test
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
  main "$@"
fi

# vim: set sw=2 et :