Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Read domains from a file and follow each using an HTTP GET request
#!/usr/bin/env bash
# -----
# Read domains from a file and test HTTP GET request.
#
set -o errtrace
set -o pipefail
set -o errexit
set -o nounset
# Set variables for current file, directory.
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
__file="${__dir}/$(basename "${BASH_SOURCE[0]}")"
# Environment variables.
COLUMNS="${COLUMNS:-120}"
LOG_LEVEL="${LOG_LEVEL:-6}"
IFS=$'\n\t'
# Script variables.
file_path="$1"
wait_time=10
do_scheme=(
"http"
"https"
)
readonly file_path
readonly wait_time
#######################################
# Error backtrace report
# Trap signal ERR
#
# Globals:
# __file
# Arguments:
# function_name
# line_number
# Returns:
# None
#######################################
__error_backtrace_report() {
local error_code
error_code=${?}
echo "Fatal error: function ${1} in ${__file} on line ${2}" 1>&2
exit ${error_code}
}
trap '__error_backtrace_report "${FUNCNAME:-.}" "${LINENO}"' ERR
col_width=0
while read -r line; do
if [[ "${#line}" -gt "$col_width" ]]; then
col_width="${#line}";
fi
done < "$file_path"
col_width=$(( col_width + 10 ))
prev_domain=""
prev_response_code=""
prev_response_location=""
while read -r domain_name; do
if [[ -z "$domain_name" ]]; then
continue
fi
for scheme in "${do_scheme[@]}"; do
http_request_url="$scheme://$domain_name";
http_response_code=$(
curl -kI -m"$wait_time" -w '%{http_code}' -s -o /dev/null "$http_request_url" || true
)
if [[ $http_response_code -ge 200 ]] && [[ $http_response_code -lt 400 ]];
then
http_response_location=$(
curl -kI -m"$wait_time" -w '%{url_effective}' -Ls -o /dev/null "$http_request_url" || true
)
else
http_response_location="-";
fi
if [[ "$prev_domain" != "$domain_name" ]] || [[ "$prev_response_code" != "$http_response_code" ]] || [[ "$prev_response_location" != "$http_response_location" ]]; then
printf -v col1 "%-*s" "$col_width" "$http_request_url"
echo "URL: $col1 Response: $http_response_code Location: $http_response_location "
fi
prev_domain="$domain_name"
prev_response_code="$http_response_code"
prev_response_location="$http_response_location"
done;
done < "$file_path"
@christopher-hopper

This comment has been minimized.

Copy link
Owner Author

christopher-hopper commented Oct 17, 2018

Usage:

domains-test.sh ~/domain-list.txt

Create a text file with one domain name per line, then call the script with the path to your text file.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.