Skip to content

Instantly share code, notes, and snippets.

@Loschcode
Last active August 15, 2016 01:10
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Loschcode/dbcc3d247d3a08c696b55e216ebaa266 to your computer and use it in GitHub Desktop.
Save Loschcode/dbcc3d247d3a08c696b55e216ebaa266 to your computer and use it in GitHub Desktop.
require IEx
defmodule Crawler do
@remote_url "http://www.iafd.com/person.rme/perfid=avaalvarez/gender=f/ava-alvarez.htm"
@datas_list [:ethnicity, :nationality, :hair, :height, :weight, :mensurations, :random]
@datas_selector "#home .biodata"
def start do
request! |> process_response
end
defp process_response(response) do
for data <- @datas_list do
response |> fetch_tags
|> select_tag(data)
|> process_result(data)
end
end
defp process_result(result, data) when is_tuple(result), do: success(data, result)
defp process_result(_, data), do: error(data)
defp error(data) do
IO.puts "Impossible to resolve #{clean_name(data)}."
end
defp success(data, result) do
IO.puts "#{clean_name(data)} is #{extract_from_tag(result)}"
end
defp select_tag(list, :ethnicity), do: hd(list)
defp select_tag(list, :nationality), do: Enum.at(list, 1)
defp select_tag(list, :hair), do: Enum.at(list, 2)
defp select_tag(list, :height), do: Enum.at(list, 3)
defp select_tag(list, :weight), do: Enum.at(list, 4)
defp select_tag(list, :mensurations), do: Enum.at(list, 2)
defp select_tag(_list, _), do: IO.puts "I don't know what you're talking about"
defp fetch_tags(response) do
Floki.find(response.body, @datas_selector)
end
defp clean_name(atom) do
to_string(atom) |> String.capitalize
end
defp extract_from_tag(tag) do
tag |> elem(2) |> hd
end
defp request! do
HTTPoison.start
HTTPoison.get! @remote_url
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment