Read domains from a file and follow each using an HTTP GET request

domains-test.sh

#!/usr/bin/env bash
# -----
# Read domains from a file and test HTTP GET request.
#

set -o errtrace
set -o pipefail
set -o errexit
set -o nounset

# Set variables for current file, directory.
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
__file="${__dir}/$(basename "${BASH_SOURCE[0]}")"

# Environment variables.
COLUMNS="${COLUMNS:-120}"
LOG_LEVEL="${LOG_LEVEL:-6}"
IFS=$'\n\t'

# Script variables.
file_path="$1"
wait_time=10
do_scheme=(
    "http"
    "https"
)

readonly file_path
readonly wait_time

#######################################
# Error backtrace report
# Trap signal ERR
#
# Globals:
#   __file
# Arguments:
#   function_name
#   line_number
# Returns:
#   None
#######################################
__error_backtrace_report() {
    local error_code
    error_code=${?}
    echo "Fatal error: function ${1} in ${__file} on line ${2}" 1>&2
    exit ${error_code}
}
trap '__error_backtrace_report "${FUNCNAME:-.}" "${LINENO}"' ERR

col_width=0
while read -r line; do
  if [[ "${#line}" -gt "$col_width" ]]; then 
      col_width="${#line}"; 
  fi
done < "$file_path"
col_width=$(( col_width + 10 ))

prev_domain=""
prev_response_code=""
prev_response_location=""

while read -r domain_name; do
    if [[ -z "$domain_name" ]]; then
        continue
    fi
    for scheme in "${do_scheme[@]}"; do
        http_request_url="$scheme://$domain_name";
        http_response_code=$(
            curl -kI -m"$wait_time" -w '%{http_code}' -s -o /dev/null "$http_request_url" || true
        )
        if [[ $http_response_code -ge 200 ]] && [[ $http_response_code -lt 400 ]];
        then
            http_response_location=$(
                curl -kI -m"$wait_time" -w '%{url_effective}' -Ls -o /dev/null "$http_request_url" || true
            )
        else
            http_response_location="-";
        fi
        if [[ "$prev_domain" != "$domain_name" ]] || [[ "$prev_response_code" != "$http_response_code" ]] || [[ "$prev_response_location" != "$http_response_location" ]]; then
            printf -v col1 "%-*s" "$col_width" "$http_request_url" 
            echo "URL: $col1 Response: $http_response_code  Location: $http_response_location " 
        fi

        prev_domain="$domain_name"
        prev_response_code="$http_response_code"
        prev_response_location="$http_response_location"
    done;
done < "$file_path"

Usage:

domains-test.sh ~/domain-list.txt

Create a text file with one domain name per line, then call the script with the path to your text file.