Gist To download File from File Directory exposed via Web Server

ruby_spider.rb

require "rubygems"
require "net/http"
require "open-uri"
require "fileutils"

##  BASE_DIR where the file will be stored
BASE_DIR = ""
def open_url(url)
  begin 
    url = File.join(BASE_PATH,url)
    puts "Fetching  url --------------> #{url}"
    raise "External domain file" if url.scan(/http/).count > 1
    open(url).read 
  rescue => e 
    puts "Got Error -- #{e} ---" "Error Fetching File Please try again later "
  end ## begin/rescue 
end ## method end

## BASE_PATH from where the url will be fetch
BASE_PATH=""
def download(parent_directory,download_arry)
  download_arry  -=  ["fckeditor/"]
  download_arry.each do |file|
    link = File.join(parent_directory,file)
    html = open_url(link)
    ## Regex to match the if the Parent Directory link is present in the file if not save else traverse
    if html.match(/<a href=\"(.+)\"> Parent Directory<\/a><\/li>/)
      #parent_directory = html.match(/<a href=\"(.+)\"> Parent Directory<\/a><\/li>/)[1]
      html.gsub!(/<a href=\"(.+)\"> Parent Directory<\/a><\/li>/,"")
      download(link,html.scan(/<a href=\"(.+)\">/).flatten.compact.uniq)
    else
      unless File.exists?(File.join(BASE_DIR,parent_directory))
        FileUtils.mkdir_p File.join(BASE_DIR,parent_directory)
      end ## unless end	
      File.open(File.join(BASE_DIR,parent_directory,file),'w+') do |file_handler|
        file_handler.write html
      end ## File closed
    end ## if/else end	
  end	
end ## method end
 
download(String.new,["/vendor/plugins"])