Download all files from IATI

download.rb

require 'httparty'

def log(target_file, resource_url, package_id) 
	File.open("downloads.log", 'a') do |file|
	  file.puts "#{DateTime.now.to_s} ::: Downloaded #{target_file} from #{resource_url} in #{package_id}"
	end	
end

Dir.mkdir("data", 0700) unless Dir.exists?("data")

package_list = HTTParty.get("http://www.iatiregistry.org/api/3/action/package_list")

package_list['result'].each do |package_id|
	package = HTTParty.get("http://www.iatiregistry.org/api/3/action/package_show?id=#{package_id}")
	package['result']['resources'].each do |resource|
		resource_url = resource['url']
		target_file  = "data/#{package_id}.xml"

		if(File.exists?(target_file)) then
			puts "File #{target_file} already exists, skipping"
		else
			begin
				puts "Downloading #{target_file} from #{resource_url}"
				File.open(target_file, "wb") do |f| 
				  f.write HTTParty.get(resource_url).body
				end	

				log(target_file, resource_url, package_id)
			rescue Exception => e
				puts "Error downloading previous file"
				puts e
			end
		end
	end
end