Skip to content

Instantly share code, notes, and snippets.

@philpennock
Last active November 20, 2023 16:34
Show Gist options
  • Save philpennock/43bb5aabd2537064f3557c4519bbb0cf to your computer and use it in GitHub Desktop.
Save philpennock/43bb5aabd2537064f3557c4519bbb0cf to your computer and use it in GitHub Desktop.
Bash script, using dig & curl, for reporting DNS and a few HTTPS policy files for everything email about a domain
#!/usr/bin/env bash
#
# Copyright 2020,2021,2022 Pennock Tech, LLC
# No warranty, this is a proof-of-concept not a final product.
# MIT-style license.
set -euo pipefail
# This might need to switch to another language for concurrency and handling
# the queries which are rarer, but this is a decent start as a proof-of-concept.
#
# I explicitly don't handle dig being old here.
# If the script needs more than this, then switch to using _lib.sh
progname="$(basename "$0" .sh)"
declare -i VERBOSE=0
warn() { printf >&2 '%s: %s\n' "$progname" "$*"; }
die() { warn "$@"; exit 1; }
# Needs to be POSIX ERE; RFC 5321 'sub-domain'; does not include underscore
readonly RE_VALID_DNSLABEL='[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
# DKIM selectors can be hierarchical
readonly RE_VALID_DKIM_SELECTOR="(${RE_VALID_DNSLABEL}(\.${RE_VALID_DNSLABEL})*)"
readonly WHITESPACE=$' \t'
# Inhibit curl with -C
readonly CURL_USER_AGENT='dns-email/0.1 (Phil Pennock, @philpennock GitHub)'
# If no DNSSEC confirmed for a domain, we will check this domain's DNSSEC status
# to know if it's the resolver or the domain.
readonly DNSSEC_KNOWN_SIGNED_DOMAIN='ripe.net'
declare -a DENY_DNSXL_SERVERS ALLOW_DNSXL_SERVERS ALLOW_DNSXL_DOMAIN_SERVERS
DENY_DNSXL_SERVERS=(
zen.spamhaus.org
dnsbl.dronebl.org
psbl.surriel.com
relays.nether.net
bl.spamcop.net
auth.spamrats.com
)
ALLOW_DNSXL_SERVERS=(
list.dnswl.org
swl.spamhaus.org
)
ALLOW_DNSXL_DOMAIN_SERVERS=(
dwl.dnswl.org
)
declare -a PKIX_CERT_TA_FILES
PKIX_CERT_TA_FILES=(
# These should contain comments making it easy to go from TLSA records
# back to cert names. We'll check them for the comments and discard any
# which are missing, so it _should_ be okay to use system certs here too
"$HOME/etc/services/x509/cert.pem"
"/etc/ssl/cert.pem"
"/usr/local/etc/ssl/cert.pem"
"/etc/ssl/certs/ca-certificates.crt"
"/etc/pki/tls/certs/ca-bundle.crt"
"/etc/ssl/ca-bundle.pem"
"/etc/pki/tls/cacert.pem"
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
)
declare -a dns_resolvers=()
usage() {
local ev="${1:-1}"
[[ $ev -eq 0 ]] || exec >&2
cat <<EOUSAGE
Usage: $progname [-CLMNTcmtvx] [-A <L>] [-D <L>] [-W <L>] [-R <R>] [-i <IP>] [-s <SEL>] domain [domain ...]
-v be more verbose (can be repeated)
-C disable curl (for MTA-STS, etc)
-M disable assumption that MX hostnames send email
-m fetch mta-sts even without DNS trigger
-N number some resources
-s SEL look for a DKIM selector <SEL> in the domain
-x enable DNSxL checks for mail-server IPs
-i add IP to list of IPs to check
-D DDL add <DDL> to list of Deny DNSxL servers (- to blank)
-A DAL add <DAL> to list of Allow DNSxL servers (- to blank)
-W DDAL add <DDAL> to list of Domain Allow DNSxL servers (- to blank)
-L use local DNS resolver instead of an upstream or -R
-t use DNS/TCP (53/tcp) for the DNS resolver
-T use DNS-over-TLS (853/tcp) for the DNS resolver
-R RES use <RES> as DNS resolver (unless -L used)
-c enable color output [also triggered by \$CLICOLOR/CLICOLOR_FORCE]
Deny DNSxL servers: ${DENY_DNSXL_SERVERS[*]}
Allow DNSxL servers: ${ALLOW_DNSXL_SERVERS[*]}
Domain Allow DNSxL servers: ${ALLOW_DNSXL_DOMAIN_SERVERS[*]}
EOUSAGE
find_diglike_command
exit "$ev"
}
main() {
local -i parse_options_caller_shift
parse_options "$@"
shift "$parse_options_caller_shift"
if [[ $# -eq 0 ]]; then
usage 1
fi
find_diglike_command
derive_dns_resolvers
local -A TODO_REPORT
init_colors
local first_output=true
local domain
for domain; do
$first_output || echo
query_domain "$domain"
first_output=false
done
# These are chained so that I can keep the to-do marker near the place where
# it belongs, but still report them to the user at the end, and only once no
# matter how many queries they make.
if [[ ${#TODO_REPORT[@]} -gt 0 ]]; then
if (( VERBOSE )); then
echo
printf '# TODO: %s\n' "${TODO_REPORT[@]}"
fi
fi
exit 0
}
# SIDE-EFFECT: sets $opt_* and $parse_options_caller_shift
# SIDE-EFFECT: updates $VERBOSE
parse_options() {
opt_assume_mx_send=true
opt_enable_curl=true
opt_enumerate=false
opt_additional_ips=()
opt_enable_dnsxl=false
opt_mta_sts_always=false
opt_res_local=false
opt_res_tcp=false
opt_res_tls=false
opt_resolvers=() # we only use one at present, but permit multiple
opt_dkim_selectors=()
opt_enable_color=false
if [[ -n "${CLICOLOR_FORCE:-}" ]] && [[ "$CLICOLOR_FORCE" != "0" ]]; then
opt_enable_color=true
elif [[ -n "${CLICOLOR:-}" ]] && [[ "$CLICOLOR" != "0" ]] && [[ -t 1 ]]; then
opt_enable_color=true
fi
if [[ -n "${NO_COLOR:-${NOCOLOR:-}}" ]]; then
opt_enable_color=false
fi
local arg OPTIND
while getopts ':hci:ms:tvxA:D:CLMNR:TW:' arg; do
case "$arg" in
h) usage 0 ;;
c) opt_enable_color=true ;;
i) opt_additional_ips+=("$OPTARG") ;;
m) opt_mta_sts_always=true ;;
s)
if [[ "$OPTARG" =~ ^${RE_VALID_DKIM_SELECTOR}$ ]]; then
opt_dkim_selectors+=("$OPTARG")
else
die "bad -s DKIM selector '${OPTARG}'"
fi
;;
t) opt_res_tcp=true ;;
v) VERBOSE+=1 ;;
x) opt_enable_dnsxl=true ;;
A) if [[ "$OPTARG" == "-" ]]; then ALLOW_DNSXL_SERVERS=(); else ALLOW_DNSXL_SERVERS+=("$OPTARG"); fi ;;
D) if [[ "$OPTARG" == "-" ]]; then DENY_DNSXL_SERVERS=(); else DENY_DNSXL_SERVERS+=("$OPTARG"); fi ;;
C) opt_enable_curl=false ;;
L) opt_res_local=true ;;
M) opt_assume_mx_send=false ;;
N) opt_enumerate=true ;;
R) if [[ "$OPTARG" == "-" ]]; then opt_resolvers=(); else opt_resolvers+=("$OPTARG"); fi ;;
T) opt_res_tls=true ;;
W) if [[ "$OPTARG" == "-" ]]; then ALLOW_DNSXL_DOMAIN_SERVERS=(); else ALLOW_DNSXL_DOMAIN_SERVERS+=("$OPTARG"); fi ;;
# not happy at letter choice but nothing saner still left
:) die "missing required option for -$OPTARG; see -h for help" ;;
\?) die "unknown option -$OPTARG; see -h for help" ;;
*) die "unhandled option -$arg; CODE BUG" ;;
esac
done
parse_options_caller_shift=$((OPTIND - 1))
shift $parse_options_caller_shift
# any extra keyword options processing here, be sure to adjust parse_options_caller_shift
if $opt_res_tcp && $opt_res_tls; then
die "-t and -T are incompatible"
fi
}
have_command() { command -v "$1" >/dev/null 2>&1 ; }
# SIDE-EFFECT: sets: $use_kdig $use_dig
find_diglike_command() {
use_kdig=0
use_dig=0
if have_command kdig; then
use_kdig=1
elif have_command dig; then
use_dig=1
else
die "fatal problem: missing a kdig or dig command"
fi
}
# SIDE-EFFECT: sets $RRDATA[@] and $RRDATA_SAW_ALIAS
# Will skip alias type records
set_rrdata() {
local data="${1?}"
# bash, -g means global; zsh, means non-local.
# We're sticking to bash, so we can't have a maskable array without just assigning it
RRDATA=()
RRDATA_SAW_ALIAS=false
if [[ "$data" == "" ]]; then
return
fi
local rrname ttl class rrtype rrdata
while read -r rrname ttl class rrtype rrdata; do
case "${rrtype^^}" in
CNAME) RRDATA_SAW_ALIAS=true ;;
DNAME) RRDATA_SAW_ALIAS=true ;;
*) RRDATA+=("$rrdata") ;;
esac
done <<<"$data"
}
# SIDE-EFFECT: sets $MX_HOSTS[@]
# SIDE-EFFECT: indirectly causes $RRDATA[@] to be set
set_mx_hosts() {
local RRDATA
set_rrdata "${1:?}"
MX_HOSTS=()
local mx
local pattern='^[[:digit:]]+[[:space:]]+([^[:space:]]+)$'
for mx in "${RRDATA[@]}"; do
if [[ $mx == "0 ." ]]; then
printf '# NULL MX, no email accepted here\n'
continue
fi
[[ $mx =~ $pattern ]] || continue
MX_HOSTS+=("${BASH_REMATCH[1]}")
derive_implicit_dkim_mx "${BASH_REMATCH[1]}"
done
}
# SIDE-EFFECT: adds to $TODO_REPORT[@]
query_domain() {
local domain="${1:?}"
local -l lower_domain="$domain"
local RRDATA MX_HOSTS MX_IPS IMPLICIT_DKIM_SELECTORS
local domain_has_apex_address
local mx_lines
local d s t saw_tlsa ip rev_ip
printf '# Domain: %s\n' "$domain"
if domain_is_signed "$domain" ; then
printf '# is DNSSEC signed\n'
elif resolver_not_verifying; then
printf '# (our resolver does not verify DNSSEC)\n'
else
printf '# is *NOT* DNSSEC signed\n'
fi
local mx_lines
mx_lines="$(dig_one -t mx "$domain")"
if [[ -z "$mx_lines" ]]; then
printf '# No MX records found??\n'
else
set_mx_hosts "$mx_lines"
printf '# MX:\n%s\n' "$mx_lines"
# This is a heuristic: we really care about the reverse DNS of the hosts
# which _send_ email _outbound_ from a domain, but we don't have a means of
# identifying those. For sufficiently small mail domains, the MX hosts will
# also be the sending hosts, so check the MX hosts for reverse DNS.
#
# This will give false positive complaints for large providers.
#
# We don't care about reverse DNS of the MX host which we connect outbound to,
# folks don't do spam checks on who they send to, only whom they receive from.
if $opt_assume_mx_send; then
if inhibit_assume_mx_send_by_mx "${MX_HOSTS[@]}"; then
printf '# Matched MX "%s", MX are not senders\n' "$INHIBIT_MATCH"
else
check_mx_hosts "$domain"
printf '# MX IP addresses: %s\n' "${MX_IPS[*]}"
fi
fi
fi # missing MX guard
printf '\n'
t="$(dig_one -t a "$domain"; dig_one -t aaaa "$domain")"
if [[ -n "$t" ]]; then
domain_has_apex_address=true
else
domain_has_apex_address=false
fi
printf '# SPF:\n'
chase_spf "$domain"
# do we want to check if the MX host is in the SPF records? Would require
# iterating (with depth/query limits) and finding CIDR blocks and checking
# various other SPF entry types.
# We should also have a "get MX_IPS" approach which we can call if not
# checking MX hosts for forward/reverse DNS.
printf '# DMARC/historical/other policies:\n'
dig_one_txt_else _dmarc "$domain" 'v=DMARC1' DMARC
t="$(dig_one -t txt "_dmarc.${domain}")" # this is the exact same check just done, we lose the shell-layer cache to avoid re-architect, but it should at least be readily available in DNS caches.
# RFC 7489 §7.1 requires that a target domain which isn't ours have authorization records.
# Without implementing PSL checks, I'm going to target the common case of unrelated domains, rather than sub-domains of a public domain.
# We can fix this logic if it triggers too many false positives because of sub-domain usage within one public domain.
local dmarc_alt_checked=''
if [[ "$t" =~ rua=mailto:([^${WHITESPACE};]+)(;.*)?$ ]]; then # FIXME: the value can be a comma-separated list, need to handle that.
d="${BASH_REMATCH[1]}"
d="${d##*@}"
if [[ "$d" != "$domain" ]]; then
dig_one_txt_else "$domain._report._dmarc" "$d" 'v=DMARC1' 'DMARC authorization'
dmarc_alt_checked="$d"
fi
fi
if [[ "$t" =~ ruf=mailto:([^${WHITESPACE};]+)(;.*)?$ ]]; then # FIXME: the value can be a comma-separated list, need to handle that.
d="${BASH_REMATCH[1]}"
d="${d##*@}"
if [[ "$d" != "$domain" ]] && [[ "$d" != "$dmarc_alt_checked" ]]; then
dig_one_txt_else "$domain._report._dmarc" "$d" 'v=DMARC1' 'DMARC authorization'
dmarc_alt_checked="$d"
fi
fi
dig_one_txt_else _adsp._domainkey "$domain" '' ADSP
dig_one_txt_else default._bimi "$domain" 'v=BIMI1' BIMI
# Common pattern
local postmaster
postmaster="$(dig_one -t a "postmaster.$domain"; dig_one -t aaaa "postmaster.$domain"; true)"
if [[ -n "$postmaster" ]]; then
printf '# postmaster exists in DNS, try <https://postmaster.%s/>\n' "$domain"
printf '%s\n' "$postmaster"
else
printf '# postmaster hostname not found in DNS\n'
fi
# draft-ietf-marid-csv-csa-01 from 2004; went nowhere as a standard AFAIK but some use it
printf '# CSA:\n'
dig_one -t srv "_client._smtp.$domain"
local -a all_selectors dedup_selectors
local -A done_selector
all_selectors=("${opt_dkim_selectors[@]}" "${IMPLICIT_DKIM_SELECTORS[@]}")
if [[ "${#all_selectors[@]}" -gt 0 ]]; then
for s in "${all_selectors[@]}"; do
if ${done_selector[$s]:-false}; then continue; fi
dedup_selectors+=("$s")
done_selector["$s"]=true
done
printf '\n# DKIM records for selectors: %s\n' "${dedup_selectors[*]}"
for s in "${dedup_selectors[@]}"; do
t="$(dig_one -t txt "${s}._domainkey.${domain}" || true)"
show_or_comment "$t" "MISSING [$s]"
done
fi
# RFC 6186 for the framework; RFC 8314 adds submissions and strongly discourages non-TLS
printf '\n# Client SRV records:\n'
for s in submissions submission imaps imap pop3s pop3 sieve; do
dig_one -t srv "_${s}._tcp.$domain"
done
# draft-koch-openpgp-webkey-service
printf '\n# User identity:\n'
wkd_check "$domain" "$domain_has_apex_address"
# RFC 8659 (with extensions in 8657)
printf '\n# TLS Policies:\n'
t="$(dig_one -t caa "$domain")"
show_or_comment "$t" "No CAA records found"
# FIXME: need an AD check for DNSSEC here
saw_tlsa=false
for h in "${MX_HOSTS[@]}"; do
t="$(dig_one -t tlsa "_25._tcp.$h")"
if [[ -n "$t" ]]; then
render_tlsa_records "$t"
saw_tlsa=true
fi
done
if ! $saw_tlsa; then
printf '# No TLSA records found\n'
fi
# RFC 8461
t="$(dig_one -t txt "_mta-sts.$domain")"
show_or_comment "$t" "No MTA-STS enabling TXT trigger found"
dig_one -t a "mta-sts.$domain"
dig_one -t aaaa "mta-sts.$domain"
local do_mtasts_fetch=false
if $opt_enable_curl && contains_rrtype TXT "$t" 'v=STSv1'; then
do_mtasts_fetch=true
elif [[ -n "$t" ]] && ! contains_rrtype TXT "$t"; then
printf '# MTA-STS enabling TXT trigger results missing TXT record (bad wildcard?)\n'
elif [[ -n "$t" ]] && ! contains_rrtype TXT "$t" 'v=STSv1'; then
printf '# MTA-STS enabling TXT trigger missing enable prefix, ignoring\n'
fi
if $opt_enable_curl && $opt_mta_sts_always && ! $do_mtasts_fetch; then
printf '# MTA-STS fetch triggered explicitly by caller anyway\n'
do_mtasts_fetch=true
fi
if "$do_mtasts_fetch"; then
t="$(cmd_curl_trace "https://mta-sts.$domain/.well-known/mta-sts.txt" || true )"
if [[ -z "$t" ]]; then
printf '# no mta-sts.txt retrieved?\n'
else
printf '%s\n' "$t" | sed 's/^/ mta-sts: /'
fi
fi
# RFC 8460: SMTP TLS Reporting
dig_one_txt_else _smtp._tls "$domain" 'v=TLSRPTv1' TLS-RPT
if $opt_enable_dnsxl; then
printf '\n'
printf '# DNSxL checks: allow-lists (presence _good_):\n'
for ip in "${MX_IPS[@]}" "${opt_additional_ips[@]}"; do
rev_ip="$(reverse_ip "$ip")" || continue
if (( VERBOSE )); then printf '# IP=[%s] => reversed IP [%s]\n' "$ip" "$rev_ip"; fi
for dnsxl in "${ALLOW_DNSXL_SERVERS[@]}"; do
if (( VERBOSE )); then printf '# DNSxL=<%s>\n' "$dnsxl"; fi
dig_one -t a "${rev_ip}.${dnsxl}"
dig_one -t txt "${rev_ip}.${dnsxl}"
done
done
printf '# DNSxL checks: domain-allow-lists (presence _good_):\n'
for dnsxl in "${ALLOW_DNSXL_DOMAIN_SERVERS[@]}"; do
if (( VERBOSE )); then printf '# DNSxL=<%s>\n' "$dnsxl"; fi
dig_one -t a "${domain}.${dnsxl}"
dig_one -t txt "${domain}.${dnsxl}"
done
printf '# DNSxL checks: deny-lists (presence **BAD**):\n'
for ip in "${MX_IPS[@]}" "${opt_additional_ips[@]}"; do
rev_ip="$(reverse_ip "$ip")" || continue
if (( VERBOSE )); then printf '# IP=[%s] => reversed IP [%s]\n' "$ip" "$rev_ip"; fi
for dnsxl in "${DENY_DNSXL_SERVERS[@]}"; do
if (( VERBOSE )); then printf '# DNSxL=<%s>\n' "$dnsxl"; fi
dig_one -t a "${rev_ip}.${dnsxl}"
dig_one -t txt "${rev_ip}.${dnsxl}"
done
done
fi
}
show_or_comment() {
local body comment returncode
body="${1-}" # empty explicitly okay
comment="${2:?}"
returncode="${3:-false}"
if [[ -n "$body" ]]; then
printf '%s\n' "$body"
if [[ "$body" =~ '\\"' ]]; then
printf '#\n# WARNING: LOOKS LIKE EMBEDDED LITERAL QUOTES IN THAT!\n#\n'
fi
if $returncode; then return 0; fi
else
printf '# %s\n' "$comment"
if $returncode; then return 1; fi
fi
}
show_or_comment_check_rrtype_starts() {
local body rrtype prefix system
body="${1-}" rrtype="${2:?}" prefix="${3:-}" system="${4:?}"
if show_or_comment "$body" "No $system $rrtype record found" true; then
if ! contains_rrtype "$rrtype" "$body" "$prefix"; then
printf '# no "%s" in a %s record, so no %s\n' "$prefix" "$rrtype" "$system"
fi
fi
}
dig_one_txt_else() {
local sublabel domain seekprefix label
sublabel="${1:?}" domain="${2:?}" seekprefix="${3:-}" label="${4:?}"
local t
t="$(dig_one -t txt "${sublabel}.${domain}")"
show_or_comment_check_rrtype_starts "$t" TXT "$seekprefix" "$label"
}
contains_rrtype() {
local want_rrtype="${1:?}"
local data="${2-}"
local prefix="${3:-}"
[[ -n "$data" ]] || return 1
local rrname ttl class rrtype rrdata
local -u needle this_rrtype
needle="$want_rrtype"
while read -r rrname ttl class rrtype rrdata; do
echo >/dev/null "shellcheck silence unused variables, named for identification not because need them: $rrname $ttl $class"
this_rrtype="$rrtype"
[[ "$this_rrtype" == "$needle" ]] || continue
if [[ -n "${prefix:-}" ]]; then
case "$rrdata" in
"$prefix"*) return 0 ;;
\""$prefix"*) return 0 ;;
*) continue ;;
esac
fi
return 0
done <<<"$data"
return 1
}
# SIDE-EFFECT: sets $MX_IPS[@]
# SIDE-EFFECT: sets $MX_CONTAINS_ALIAS
check_mx_hosts () {
local domain="${1:?}"
local RRDATA REVERSE_ERRORS REVERSE_WARNINGS PTR_BLOCKS
local addr_lines_a addr_lines_aaaa one_mx ADDRS
# Deliberate leak; we use it here, but let callers see it too
MX_CONTAINS_ALIAS=false
printf '# MX hosts:\n'
ADDRS=()
REVERSE_ERRORS=()
REVERSE_WARNINGS=()
PTR_BLOCKS=()
declare -A checked_mx_hosts
for one_mx in "${MX_HOSTS[@]}"; do
if [[ "${checked_mx_hosts[$one_mx]:+yes}" == "yes" ]]; then
if (( VERBOSE )); then
printf '## [%s] SKIPPING duplicate MX hostname: %s\n' "$domain" "$one_mx"
fi
continue
fi
checked_mx_hosts[$one_mx]=t
addr_lines_a="$(dig_one -t a "$one_mx")"
addr_lines_aaaa="$(dig_one -t aaaa "$one_mx")"
printf '%s\n' "$addr_lines_a" "$addr_lines_aaaa"
set_rrdata "$addr_lines_a"
if "$RRDATA_SAW_ALIAS"; then MX_CONTAINS_ALIAS=true; fi
ADDRS+=("${RRDATA[@]}")
check_mxhost_reverse_dns "$one_mx" A "${RRDATA[@]}"
set_rrdata "$addr_lines_aaaa"
ADDRS+=("${RRDATA[@]}")
check_mxhost_reverse_dns "$one_mx" AAAA "${RRDATA[@]}"
done
if $MX_CONTAINS_ALIAS; then
printf '#\n%s# *** ERROR: MX records contained an alias rrtype ***%s\n#\n' "$COLOR_START_ERR" "$COLOR_END"
fi
echo
printf '# MX Reverse DNS:\n'
printf '%s\n' "${PTR_BLOCKS[@]}"
if [[ ${#REVERSE_ERRORS[@]} -eq 0 ]] && [[ ${#REVERSE_WARNINGS[@]} -eq 0 ]]; then
printf '# no problems with reverse DNS\n'
else
if [[ ${#REVERSE_ERRORS[@]} -gt 0 ]]; then
printf '#\n# Reverse DNS errors:%s\n' "$COLOR_START_ERR"
printf '#ERROR: %s\n' "${REVERSE_ERRORS[@]}"
printf '%s' "$COLOR_END"
fi
if [[ ${#REVERSE_WARNINGS[@]} -gt 0 ]]; then
printf '#\n# Reverse DNS minor issues:%s\n' "$COLOR_START_WARN"
printf '#warning: %s\n' "${REVERSE_WARNINGS[@]}"
printf '%s' "$COLOR_END"
fi
fi
MX_IPS=("${ADDRS[@]}")
}
# SIDE-EFFECT: appends to $REVERSE_ERRORS[@]
# SIDE-EFFECT: appends to $REVERSE_WARNINGS[@]
# SIDE-EFFECT: appends to $PTR_BLOCKS[@]
check_mxhost_reverse_dns() {
local mxhost="${1:?}"
local rrtype="${2:?}"
shift 2
local RRDATA
if [[ $# -eq 0 ]]; then
REVERSE_ERRORS+=("[$mxhost] missing forward $rrtype DNS")
return
fi
local addr ptr_lines missing_mxhost entry
for addr; do
ptr_lines="$(dig_one -x "$addr")"
if [[ -z "$ptr_lines" ]]; then
REVERSE_ERRORS+=("[$mxhost] no reverse DNS for address: $addr")
continue
fi
PTR_BLOCKS+=("$ptr_lines")
set_rrdata "$ptr_lines"
# CNAMEs are definitely okay here, that's how non-octet reverse delegation is done
local -a FOUND_PTRS
FOUND_PTRS=("${RRDATA[@]}")
missing_mxhost=true
for entry in "${FOUND_PTRS[@]}"; do
if [[ "$entry" == "$mxhost" ]]; then
missing_mxhost=false
break
elif (( VERBOSE >= 2 )); then
printf '### [%s] no-match: seeking %s in: %s (from %s)\n' "$mxhost" "$mxhost" "$entry" "$addr"
fi
done
if $missing_mxhost; then
if (( VERBOSE )); then
# We only really care about hostnames which can be seen from the IP
# address, which is what servers receiving an inbound MX connection see
# as their starting point. So having an MX hostname have reverse DNS
# which matches the hostname used to connect inbound to the domain is
# not something anyone ever truly checks.
REVERSE_WARNINGS+=("[$mxhost] (minor non-issue) reverse DNS missing MX hostname: $addr")
fi
local forward_lines forward_ip forward_entry
local missing_match_forward=true
for forward_entry in "${FOUND_PTRS[@]}"; do
forward_lines="$(dig_one -t "$rrtype" "$forward_entry")"
if [[ -n "$forward_lines" ]]; then
PTR_BLOCKS+=("$forward_lines") # I know, not PTRs
set_rrdata "$forward_lines"
for forward_ip in "${RRDATA[@]}"; do
if [[ "$forward_ip" == "$addr" ]]; then
missing_match_forward=false
break
elif (( VERBOSE >= 2 )); then
printf '### [%s] no-match: seeking %s in %s (from %s)\n' "$mxhost" "$addr" "$forward_ip" "$forward_entry"
fi
done
elif (( VERBOSE )); then
printf '### [%s] reverse DNS from %s has host %s which has no %s records\n' "$mxhost" "$addr" "$forward_entry" "$rrtype"
fi # have forward DNS for a ptr
done
if $missing_match_forward; then
REVERSE_ERRORS+=("[$mxhost] address $addr lacks matching forward DNS")
fi
fi
done
}
# SIDE-EFFECT: sets $INHIBIT_MATCH
inhibit_assume_mx_send_by_mx() {
local x
local -l lx
for x; do
lx="$x"
case "$lx" in
aspmx.l.google.com.) INHIBIT_MATCH="$x"; return 0 ;;
gmr-smtp-in.l.google.com.) INHIBIT_MATCH="$x"; return 0 ;; # email forwarding servers
in*-smtp.messagingengine.com.) INHIBIT_MATCH="$x"; return 0 ;;
esac
done
return 1
}
# SIDE-EFFECT: appends to $IMPLICIT_DKIM_SELECTORS[@]
chase_spf() {
local domain="${1:?}"
shift
local -i SPF_COUNT=0 SPF_DNS_QCOUNT=0 SPF_NEST_DEPTH=0
local -A CHECKED_SPF
local SPF_ABORT=false
# The limit of 10 is supposed to be for any terms which cause DNS queries,
# and it's a query limit, not a recursion limit, so these:
# mechanisms: include, a, mx, ptr, exists
# modifiers: redirect
# all count towards the limit.
# Here, SPF_COUNT is how many SPF records we see, SPF_DNS_QCOUNT is how many
# DNS records are looked up.
# Spam Assassin uses a limit of 20 because of routine violations, see
# <https://bz.apache.org/SpamAssassin/show_bug.cgi?id=7182>
# Find real current value in source, lib/Mail/SpamAssassin/Plugin/SPF.pm
# sub _check_spf, around line 530, `max_dns_interactive_terms` value
# rspamd has a total limit and a _nesting_ limit.
# src/libserver/spf.h : SPF_MAX_NESTING=10 SPF_MAX_DNS_REQUESTS=30 (defaults, can be overruled)
local -i SPF_QCOUNT_WARN=10
local SPF_WARN_MSG='should be at most 10'
SPF_DNS_QCOUNT+=1
dig_one_spf_domain "$domain"
TODO_REPORT['spf_chasing']="SPF: handle exp=<explain> expandos"
TODO_REPORT['spf_quotes']="SPF: remove entirely space between quotes, better handle multi-string"
}
dig_one() {
local cmd
local -a flags at_res
flags=( +nodnssec +nonsid +nomultiline +noall +answer )
if (( use_kdig )); then
cmd=kdig
flags+=( +nocomments )
if $opt_res_tls; then
flags+=( +tls )
fi
fi
if (( use_dig )); then
cmd=dig
flags+=( +nosearch )
if $opt_res_tls; then
die "can't use +tls with dig(1), sorry"
fi
fi
if $opt_res_tcp; then
flags+=( +tcp )
fi
if [[ "${#dns_resolvers[@]}" -ge 1 ]]; then
at_res+=("@${dns_resolvers[0]}")
fi
# echo >&2 "running: $cmd ${at_res[*]} ${flags[*]} $*"
command "$cmd" "${at_res[@]}" "${flags[@]}" "$@"
}
# SIDE-EFFECT: modifies $SPF_COUNT, $CHECKED_SPF[@], $SPF_ABORT, $IMPLICIT_DKIM_SELECTORS[@]
dig_one_spf_domain() {
local domain="${1:?}"
if [[ "${CHECKED_SPF[$domain]:+yes}" == "yes" ]]; then
# allow for diamond includes, don't abort (but could usefully look for loops)
$SPF_ABORT || printf '# Not checking twice for SPF: %s\n' "$domain"
return
fi
local -i SPF_NEST_DEPTH="$SPF_NEST_DEPTH"
SPF_NEST_DEPTH+=1
derive_implicit_dkim_spfdomain "$domain"
CHECKED_SPF[$domain]='pre'
SPF_COUNT+=1
if $SPF_ABORT; then return; fi
if (( SPF_DNS_QCOUNT > SPF_QCOUNT_WARN )); then
# nb: there are often A+AAAA lookups, but they count as just 1.
printf '%s# ERROR: too many [%d] DNS lookups for SPF, %s%s\n' "$COLOR_START_ERR" "$SPF_DNS_QCOUNT" "$SPF_WARN_MSG" "$COLOR_END"
if (( SPF_QCOUNT_WARN == 10 )); then
SPF_QCOUNT_WARN=20
SPF_WARN_MSG="exceeded SpamAssassin's limit"
elif (( SPF_QCOUNT_WARN == 20 )); then
SPF_QCOUNT_WARN=30
SPF_WARN_MSG="exceeded rspamd total query limit"
elif (( SPF_QCOUNT_WARN == 30 )); then
SPF_QCOUNT_WARN=99999
SPF_WARN_MSG='wth?'
fi
fi
if (( SPF_NEST_DEPTH == 10 )); then
# we can hit this multiple times on different branches, but should only report once per branch.
printf '%s# ERROR: exceeded rspamd nesting depth limit [%d]%s\n' "$COLOR_START_ERR" "$SPF_NEST_DEPTH" "$COLOR_END"
fi
if (( SPF_COUNT >= 100 )); then
SPF_ABORT=true
printf '# ... and that makes 100, too many to keep reporting for diagnosis\n'
return
fi
local out fout part lpart last
local -l lpart last
while read -r out; do
[[ -n "$out" ]] || continue
fout="${out//\"/}"
# shellcheck disable=SC2086
set -- $fout
shift 4 # rrname, ttl, class, rrtype
case "$1" in
v=spf1*)
if $opt_enumerate; then printf '%3d/%3d ' "$SPF_COUNT" "$SPF_DNS_QCOUNT"; fi
printf '%s\n' "$out"
;;
*)
if (( VERBOSE >= 3 )); then printf '# excluded: %s\n' "$out"; fi
continue
;;
esac
for part; do
lpart="$part" # auto-lowered
last="$part"
case "$lpart" in
include:*) SPF_DNS_QCOUNT+=1; dig_one_spf_domain "${lpart#include:}" ;;
a | a:* | mx | mx:* | ptr | ptr:*) SPF_DNS_QCOUNT+=1 ;;
exists:* ) SPF_DNS_QCOUNT+=1 ;;
esac
done
case "$last" in
redirect=*) SPF_DNS_QCOUNT+=1; dig_one_spf_domain "${last#redirect=}" ;;
esac
done <<<"$(dig_one -t txt "$domain")"
CHECKED_SPF[$domain]='done'
}
# SIDE-EFFECT: appends to $IMPLICIT_DKIM_SELECTORS[@]
derive_implicit_dkim_spfdomain() {
# This one is less useful and mostly a relic of some past practices which are dying out.
# SPF is for the SMTP Envelope Sender, which needs to point back to that mail sending service for bounces, so the only SPF records which could would be them.
# *If* you pattern-match in your MTA and route some bounce messages back to the sending service then this still helps, but that's fairly rare.
# For a while, folks were including records for all outsourced senders in their own SPF record. It's normally pointless though.
# Eg, mailchimp have stopped providing an interface to walk people through SPF setup. They're DKIM-only.
# So:
# 1. For the MX case, that's covered in another function
# 2. For some rare cases where folks align the sender and forward bounces, this does help
# 3. This might also help with vanity MX records?
# 4. For common case, this is unlikely to do too much
# Still, it's all just hints, so we might as well take the hints.
local domain="${1:?}"
case "$domain" in
_spf.google.com) IMPLICIT_DKIM_SELECTORS+=(google) ;;
mail.zendesk.com) IMPLICIT_DKIM_SELECTORS+=(zendesk1 zendesk2) ;;
servers.mcsv.net) IMPLICIT_DKIM_SELECTORS+=(k1 k2 k3) ;; # mailchimp
mktomail.com) IMPLICIT_DKIM_SELECTORS+=(m1) ;; # marketo.com
spf.protection.outlook.com | spf.protection.outlook.de ) IMPLICIT_DKIM_SELECTORS+=(selector1 selector2) ;;
spf.messagingengine.com) IMPLICIT_DKIM_SELECTORS+=(fm1 fm2 fm3) ;; # fastmail
sendgrid.net) IMPLICIT_DKIM_SELECTORS+=(s1 s2) ;; # sendgrid defaults to s1/s2 but allows up to 3 chars of custom selector
spf.mandrillapp.com) IMPLICIT_DKIM_SELECTORS+=(mandrill) ;;
spf.zoho.com | zoho.com) IMPLICIT_DKIM_SELECTORS+=(zoho) ;; # not forced, it's a walkthrough, but they suggest this and I suspect most go along with that.
*cust.gandi.net) IMPLICIT_DKIM_SELECTORS+=(gm1 gm2 gm3) ;;
# ADDING: SHOULD NOT HAVE TRAILING DOT ON MATCHER
# amazonses.com) ;; # long random selectors, nothing implicit
# mailgun.org) ;; # user-selectable
# _spf.salesforce.com) ;; # user-selectable
esac
}
# SIDE-EFFECT: appends to $IMPLICIT_DKIM_SELECTORS[@]
derive_implicit_dkim_mx() {
local mxhost="${1:?}"
case "$mxhost" in
aspmx.l.google.com.) IMPLICIT_DKIM_SELECTORS+=(google) ;;
in[12]-smtp.messagingengine.com.) IMPLICIT_DKIM_SELECTORS+=(fm1 fm2 fm3) ;; # fastmail
*.mail.gandi.net.) IMPLICIT_DKIM_SELECTORS+=(gm1 gm2 gm3) ;;
# ADDING: REQUIRES TRAILING DOT ON MATCHER
esac
}
wkd_check() {
local domain="${1:?}"
local has_apex_address="${2:?}"
local t u
t="$(dig_one -t a "openpgpkey.$domain"; dig_one -t aaaa "openpgpkey.$domain")"
printf '%s\n' "${t:-# No openpgpkey.$domain found}"
$opt_enable_curl || return 0
if [[ -n "$t" ]]; then
u="https://openpgpkey.$domain/.well-known/openpgpkey/$lower_domain/policy"
elif $has_apex_address; then
u="https://$domain/.well-known/openpgpkey/policy"
else
return 0
fi
# Ideally we'd have curl print each URL tried, as it tries them, along a location redirect chain
if t="$(cmd_curl --max-redirs 5 -L --max-time 5 "$u")"; then
printf '# <%s>\n' "$u"
if [[ -n "$t" ]]; then
printf '%s\n' "$t" | sed -e 's/^/ wkd-policy: /' -e $'16c \\\n [TRUNCATED]' -e '17{x;q}'
else
printf '# wkd policy exists and is empty (which is fine and reasonable)\n'
fi
else
printf '# no WKD policy found (tried <%s>) [%s]\n' "$u" "$?"
fi
}
render_tlsa_records() {
local raw_records="${1:?}"
local -r tlsa_pat='\bTLSA[[:space:]]+([[:digit:]]+)[[:space:]]+([[:digit:]]+)[[:space:]]+([[:digit:]]+)[[:space:]]+(.+)$'
local tlsa_usage tlsa_selector tlsa_matching tlsa_cadata
local tafile known name
local -a have_tafiles=()
# Check to see which of our augmented trust-anchor files exist
# If many do (unlikely! typically 1, or 2 if my own is present)
# then we should expect this file to be in buffer cache most of the time, so
# a grep should be fast. Thus I'm not bothering to optimize this out of the
# loop, to ensure we only filter once if invoked against multiple domains.
for tafile in "${PKIX_CERT_TA_FILES[@]}"; do
[[ -f "$tafile" ]] || continue
grep -q '^# TLSA' -- "$tafile" || continue
have_tafiles+=("$tafile")
done
if [[ "${#have_tafiles[@]}" -eq 0 ]]; then
# None have our TLSA comments letting us index-scan by what we found in DNS.
printf '%s\n' "$raw_records"
return
fi
printf '%s\n' "$raw_records" | while read -r line; do
known='' name=''
if [[ "$line" =~ $tlsa_pat ]]; then
tlsa_usage="${BASH_REMATCH[1]}"
tlsa_selector="${BASH_REMATCH[2]}"
tlsa_matching="${BASH_REMATCH[3]}"
tlsa_cadata="${BASH_REMATCH[4]}"
tlsa_cadata="${tlsa_cadata// /}"
echo >/dev/null "shellcheck silence unused variables, named for identification not because need them: $tlsa_selector $tlsa_matching"
case "$tlsa_usage" in
0) ;; # PKIX-TA
2) ;; # DANE-TA
*) printf '%s\n' "$line"; continue ;;
esac
# Anything here is theoretically a trust-anchor.
for tafile in "${have_tafiles[@]}"; do
[[ -f "$tafile" ]] || continue
known="$(grep -FiC5 -m1 "$tlsa_cadata" -- "$tafile")" || continue
break
done
if [[ -z "$known" ]]; then
printf '%s\n' "$line"
continue
fi
name="$(sed -En 's/# *Name *: *(.+)/\1/p' <<<"$known")"
if [[ -z "$name" ]]; then
printf '%s\n' "$line"
else
printf '%s ; %s\n' "$line" "$name"
fi
else
printf '%s\n' "$line"
fi
done
}
reverse_ip() {
local ip="$1"
local q1 q2 q3 q4
shift
if [[ $ip =~ ^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then
q1=$((0 + BASH_REMATCH[1]))
q2=$((0 + BASH_REMATCH[2]))
q3=$((0 + BASH_REMATCH[3]))
q4=$((0 + BASH_REMATCH[4]))
printf '%s.%s.%s.%s\n' "$q4" "$q3" "$q2" "$q1"
return 0
fi
return 1
}
cmd_curl() {
command curl \
-H "User-Agent: $CURL_USER_AGENT" \
--max-time 5 \
-fSs "$@"
}
cmd_curl_trace() {
local x
for x; do true; done
printf '# <%s>\n' "$x"
cmd_curl "$@"
}
# unused(?) but keeping around
dig_each() {
local -a params
while [[ "$1" != '--' ]]; do
params+=("$1")
shift
done
shift
local x
for x; do
dig_one "${params[@]}" "$x"
done
}
# unused(?) but keeping around
dig_one_prefixfilter() {
local filter="${1:?}"
shift
local out fout
dig_one "$@" | while read -r out; do
fout="${out//\"/}"
# shellcheck disable=SC2086
set -- $fout
case $5 in
${filter}*) printf '%s\n' "$out" ;;
*) if (( VERBOSE >= 3 )); then printf '# excluded: %s\n' "$out"; fi ;;
esac
done
}
domain_is_signed() {
local domain="${1:?}"
local header_opt res flags
if (( use_kdig )); then
header_opt='+header'
else
header_opt='+comments'
fi
res="$(dig_one "$header_opt" +adflag -t mx "$domain")"
# dig emits 'flags', kdig emits 'Flags'
flags="$(sed -En 's/^;; [Ff]lags: ([^;]+);.*/\1/p' <<<"$res")"
grep -qiE '\bad\b' <<<"$flags"
}
# SIDE-EFFECT: updates $cached_resolver_not_verifying_rv
resolver_not_verifying() {
if [[ -n "${cached_resolver_not_verifying_rv:-}" ]]; then
return "$cached_resolver_not_verifying_rv"
fi
if domain_is_signed "$DNSSEC_KNOWN_SIGNED_DOMAIN"; then
# domain is signed, so our negate sense predicate is false, so set non-zero
cached_resolver_not_verifying_rv=1
else
cached_resolver_not_verifying_rv=0
fi
return "$cached_resolver_not_verifying_rv"
}
derive_dns_resolvers() {
# This is a script of mine which finds real DNS resolvers
# instead of 127.0.0.53
if ! $opt_res_local; then
if [[ "${#opt_resolvers[@]}" -gt 0 ]]; then
dns_resolvers=("${opt_resolvers[@]}")
else
# I know the exact text format used here, so:
# shellcheck disable=SC2207
dns_resolvers=($( dns-resolvers 2>/dev/null )) || true
fi
if (( VERBOSE )); then
if (( "${#dns_resolvers[@]}" )); then
printf '# [using DNS resolver: %s]\n' "${dns_resolvers[0]}"
else
printf '# using system DNS resolver\n'
fi
fi
fi
}
# SIDE-EFFECT: sets variables starting COLOR_
init_colors() {
if $opt_enable_color; then
declare -gr COLOR_START_ERR=$'\033[31;1m' COLOR_START_WARN=$'\033[35;1m' COLOR_END=$'\033[m'
else
declare -gr COLOR_START_ERR='' COLOR_START_WARN='' COLOR_END=''
fi
}
# Don't run main if sourced; easier to test
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
main "$@"
fi
# vim: set sw=2 et :
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment