Scraper class (for jobs)

Rakefile

#!/usr/bin/env rake
require "resque/tasks"

task 'resque:setup' do
  require_relative 'scraper.rb'
end

scraper.rb

require 'resque'
require 'resque/plugins/lock'

class Scraper
  extend Resque::Plugins::Lock

  @queue = :url_serve

  #def self.lock(host, url)
  #  host
  #end

  def self.perform(host, url)
    File.open('scraper.log', 'a') do |f|  
      f.write(Time.now.to_s+" "+url+"\n")
    end
    return url
  end
end

start.rb

require_relative 'scraper.rb'

requests = [
  {
    :host => "bartolsthoorn.nl",
    :url => "http://bartolsthoorn.nl"
  },
  {
    :host => "bartlab.nl",
    :url => "http://bartlab.nl"
  },
  {
    :host => "bartolsthoorn.nl",
    :url => "http://bartolsthoorn.nl/contact"
  },
  {
    :host => "bartlab.nl",
    :url => "http://bartlab.nl/contact" 
  }
]

#Does not work:
#Resque.redis = 'localhost:6379'

requests.each do |request|
  #Equivalent: Scraper.perform request[:host], request[:url]
  Resque.enqueue(Scraper, request[:host], request[:url])
end