Test if a list of files (URLS) exist on a web server

test-urls.sh

#!/bin/bash

# List missing files from a web server. Missing means we get a non-200 response.

BASE="http://www.example.com"
FILES="index.html
about.html"

for ITEM in ${FILES}; do
  URL="${BASE}/${ITEM}"
  # --head may work in some environments
  curl -s -D - "${URL}" -o /dev/null | head -1 | grep 'HTTP/1.[01] 200' > /dev/null
  if [ $? == 1 ]; then
    echo "${ITEM} not found"
  fi
done