high_level_worker.ex

CrawlerExample.Worker do
  alias CrawlerExample.Queue
  
  def work(depth_limit \\ 5) do
    case Queue.pop() do
      :empty ->
        :ok
      {:value, [url, depth]} ->
        unless depth > depth_limit do
          case request_page(url) do
            #You can add more robust error handling here, typically If the error is an http error
            #then it means the url is likely not crawlable and not worth retrying but
            #I usually break it out by code and atleast log the specific error.
            {:error, _} ->
              :ok
            {:ok, body} ->
              get_children_urls(url, body)
              |> Enum.map(fn c_url ->
                Queue.push([depth+1, c_url])
              end)
              :ok
          end
        else
          :ok
        end
    end
  end
end