railscasts_crawler.rb

# file: railscasts.rb
# Author : Akshay Gupta
# First check you have all gems installed. Place the script in /lib folder and run the script.
# I don't have expertize in ruby, please let me know how it can be improved.
# change the path accordingly, where you want to save path
# My working env is on MacOS, one need to make some changes if running on Windows

require 'rubygems'
require 'hpricot'
require 'open-uri'
require 'logger'
$log = Logger.new('log/railscasts.log')
$path = "/Users/akshaygupta/railsvideo/railscasts/"
$stop = false


class Railscasts
attr_accessor :url

def initialize
  @@page = 1
  @@url  = "http://railscasts.com/episodes?page="
  start
end

def url
  @url = @@url+@@page.to_s
end

def start
  url
  build_doc
  screencasts_links
  download_screencasts
  next_page
  if !$stop
    start
  else
    puts "Successfully done :) Enjy all the screencasts"
  end
end

def build_doc
  begin
    $log.info("*********Fetching #{@url}***********")
    @doc = Hpricot(open(@url))
  rescue Exception => e
    $log.debug("Problem fetching #{e}")
  end
end


def screencasts_links
  begin
    @download_links = 
      (@doc/".download/a[1]").collect {|a| (a.search("[@href]").first[:href])}
    $log.info(" All Download links on this page :\n #{@download_links}")
  rescue
    $log.info("Problem in download links")
  end
end

def download_screencasts
  @download_links.each do |mov|
    begin
      file = mov.split('/').last
      res = `cd #{$path}; ls | grep "#{file}"`
      if !res
        $log.info("Now downloading file #{file}")
        result = `cd #{$path}; wget "#{mov}"`
        if result
          $log.info("Successfully Downloaded #{file}")
        end
      else
        $log.info("Already downloaded #{file}")
      end
    rescue Exception => e
      $log.info("problem downloding file #{e}")
    end
  end
end

 
  def next_page
    if @@page < 17
      @@page += 1
    else
      $log.info("All screencasts downloaded :-), Mission accomplished!!")
      $stop = true
    end
  end
end