spider.rb

require 'rubygems'
require 'net/ftp'
require 'peach'

PROTO = 'ftp'
URL = 'ladsweb.nascom.nasa.gov'
PATH = 'allData/5/'
DATASET = 'MOD03'
YEARS = 2000..2011
WORKPATH = "#{PATH}#{DATASET}"

main_ftp = Net::FTP.new(URL)
main_ftp.passive = true
main_ftp.login
main_ftp.chdir(WORKPATH)



YEARS.each do |year|
	puts year.to_s
	main_ftp.chdir(year.to_s)
	days = main_ftp.list('*')

	days.peach(3) do |day|
    day = /\d{3}$/.match(day)    
    f = Net::FTP.new(URL)
    f.passive = true
    f.login
    f.chdir("#{WORKPATH}/#{year}/#{day}")
    seed_list = File.open("test/#{DATASET}_#{year}-#{day}_seeds.txt", 'w')		
		files = f.list('*.hdf')
		files.each do |file|
			file = /\d{2}\s(\w{3}\d{2}.A\d{7}.\d{4}.\d{3}.\d{13}.hdf)/.match(file)
			seed_list << "#{year.to_s}, #{day.to_s}, #{file[0].to_s}\n"
		end
		f.close
    seed_list.close
	end
  main_ftp.chdir('..')
end
main_ftp.close