wallbase.cc scraper

scraper.rb

require 'typhoeus'
require 'nokogiri'
require 'uri'

Typhoeus::Config.verbose = true

response = Typhoeus.get("http://wallbase.cc/toplist/index/?ts=1", headers: { 'X-Requested-With' => 'XMLHttpRequest' })

def download_wallpaper(wallpaper_url)
  puts "Downloading #{wallpaper_url}"

  response = Typhoeus.get(wallpaper_url)

  if response.success?
    file_name = File.basename(wallpaper_url)
    downloaded_file = File.open "wallpapers/#{file_name}", 'wb'
    downloaded_file.write(response.body)
    downloaded_file.close
  end

  response.success?
end

doc = Nokogiri::HTML(response.body)
images = doc.css '.thumbnail'
images.each do |image|
  tags = image['data-tags'].split('||').map { |t| t.split('|').first.downcase }
  thumbnail_url = image.css('.file').first['data-original']
  wallpaper_url = thumbnail_url.gsub(/thumb/, 'wallpaper')

  status = download_wallpaper(wallpaper_url)
  status ||= download_wallpaper(wallpaper_url.gsub(/jpg$/, 'png'))
end