A quick script to ping all of the site on a sitemap and check status codes

ping_sites.ex

defmodule Crawl do
  def process_resp({:ok, %{body: xml}}) do
    XmlToMap.naive_map(xml)
  end

  def get_urls(xml) do
    xml
    |> Map.get("urlset")
    |> Map.get("url")
    |> Enum.map(&Map.get(&1, "loc"))
  end

  def update_url(url) do
    url
    |> String.replace("https://developer.newrelic.com", "localhost:8000")
    |> Kernel.<>("/")
  end

  def get_page_status(url) do
    url
    |> HTTPoison.get()
    |> case do
      {:ok, %{status_code: code}} -> {code, url}
      _ -> {:error, url}
    end
  end

  def run do
    "http://localhost:8000/sitemap.xml"
    |> HTTPoison.get()
    |> process_resp()
    |> get_urls()
    |> Enum.map(&update_url/1)
    |> Enum.take(5) # For testing, just try the first 5
    |> Task.async_stream(&get_page_status/1, max_concurrency: 5)
    |> Enum.map(fn {:ok, result} -> result end)
  end
end