Load all urls from a sitemap.xml file

load_all_sitemap_urls.sh

#!/bin/bash
# This script crawls all urls in a /sitemap.xml file and loads them, effectively priming the cache
# Usage: ./warm_cache.sh www.example.com

time wget --quiet https://$1/sitemap.xml --output-document - | \
    egrep -o "https?://[^<]+" | \
    grep $1 | \
    grep -v "jpg" | \
    xargs -i -d '\n' curl --output /dev/null --silent --write-out '%{http_code} %{time_total}ms %{url_effective} \n' {}