url_checker.sh

#!/usr/bin/env bash

set -ue

export urls=( \
  # BOSH release download
  https://bosh.io \
  # PCF API with metadata for upgrades
  https://network.pivotal.io \
  # PCF-Pipelines open-source updates
  https://github.com \
  # Amazon CloudFront Load Balancer for S3
  https://d26ekeud912fhb.cloudfront.net \
  # Amazon S3 for PCF tiles and stemcells
  https://s3.amazonaws.com \
  https://s3-eu-west-1.amazonaws.com \
  # https://s3-eu-west-1.amazonaws.com \ # Not currently honoured by policy. TODO: Check if really needed
  # ADDITIONAL ENTRIES FOR DOCKER HUB (Source: https://forums.docker.com/t/list-of-docker-hub-mirror-sites-to-configure-proxy-whitelist/20845)
  # https://dseasb33srnrn.cloudfront.net \ # Returns 403 even without proxy so ignoring for the moment. TODO: Check if really needed
  https://auth.docker.io \
  https://elb-registry.us-east-1.aws.dckr.io \
  https://us-east-1-elbregis-10fucsvj1tcgy-133821800.us-east-1.elb.amazonaws.com \
  https://registry-1.docker.io \
  https://registry-origin.docker.io \
  https://index.docker.io/ \
  https://elb-io.us-east-1.aws.dckr.io \
  https://us-east-1-elbio-rm5bon1qaeo4-623296237.us-east-1.elb.amazonaws.com \
  https://registry.hub.docker.com \
  https://hub.docker.com/ \
  # ADDITIONAL ENTRIES FOR ANSIBLE
  # CF CLI download
  https://packages.cloudfoundry.org \
  #Ruby Gems Package Repository CDN
  #https://rubygems.org \
  #Debian Package Repositories
  http://deb.debian.org \
  http://security.debian.org \
  http://cdn-fastly.deb.debian.org \
  http://stedolan.github.io \
  http://starkandwayne.com \
  https://raw.githubusercontent.com
)

response_codes=()
for i in ${urls[@]}
do
  echo "[TEST] Can curl $i"
  response_code=$(curl -I $i -w '%{http_code}' -sSLko /dev/null)
  response_codes+=("$response_code")
done
echo "Results with response codes.."
echo "NOTE: 403 is considered a failure as it is likely this came from the proxy policy denying the url"
for i in "${!urls[@]}"; do
  printf "%s\t%s\n" "${response_codes[$i]}" "${urls[$i]}"
done
bad_response_codes=()
for i in "${!urls[@]}"; do
  # Amazon expected to return '405 - Method Not Allowed'
  if [[ ${response_codes[$i]} -eq 403 ]]; then
    bad_response_codes+=("${urls[$i]}")
    echo "Bad response found for ${urls[$i]}"
    set +e
    curl -sSLk -D - ${urls[$i]} | grep policy --no-messages
    set -e
    echo "For more details run 'curl -sSLk -D - ${urls[$i]}' from the same machine this script was run"
  fi
done
if [ ${#bad_response_codes[@]} -eq 0 ]; then
    echo "Success!"
else
    echo "Test failed. See above for details"
    exit 1
fi