vrybas/data_grabbing.rb

## data_grabbing.rb
# -------------------- Taking a peron name from E-mail header

  # === Returns Hash with firstname, lastname, and email from String:
  #
  # String "John Smith < john@acme.com >" will produce hash pairs:
  #
  #    :firstname => "John",
  #    :lastname => "Smith",
  #    :email => "john@acme.com"
  #
  # String "< john@acme.com >" or "john@acme.com" will produce hash pairs:
  #
  #    :firstname => "John",
  #    :lastname => nil,
  #    :email => "john@acme.com"
  def parse_mailfrom_field(from_field)
    sender = {}
    sender[:firstname] = from_field.scan(/((^)[<A-Za-z]+[^@\s])/).to_s.scan(/[A-Za-z]+/).to_s.capitalize
    sender[:lastname] = from_field.scan(/(\s[A-Za-z]+[^\s<])/).to_s[1..-1]
    sender[:email] = from_field.scan(/([^< \A]+@[^> $]+)/).to_s
    sender
  end

# -------------------- Preparing an HTML e-mail to be shown on website
  def strip_fields
    deal = self.class.find(id)
    deal.from = deal.from.scan(/(([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,}))/i).flatten.first.downcase
    stripped_body = deal.body.split('<html>') #deleting everything before <html> tag
    deal.body = '<html>'+stripped_body[1] unless stripped_body[1].blank?
    deal.save
  end


# -------------------- Parser class to get HTML data and parse it

class Parser
  require 'hpricot'
  require 'typhoeus'
  require 'open-uri'
  include Typhoeus
  define_remote_method :get_data, :timeout => 1000, :cache_responses => 3600 # 1 hour cache

  def self.currency(day = Time.now(), curr_array =['USD', 'EUR'])
    values = parse("http://www.cbr.ru/scripts/XML_daily.asp?date_req=#{day.strftime("%d/%m/%Y")}", [:Value, :CharCode, :Name], 'Valute' )
    yesterday_values = parse("http://www.cbr.ru/scripts/XML_daily.asp?date_req=#{(day-1.day).strftime("%d/%m/%Y")}", [:Value, :CharCode, :Name], 'Valute' )
    unless values.blank? && yesterday_values.blank?
      curr_today = get_currency_values(curr_array, values)
      curr_yesterday = get_currency_values(curr_array, yesterday_values)
      result = Array.new()
      curr_array.each do |curr|
        result << curr_today[curr]
        result << format_currency(curr_today[curr] - curr_yesterday[curr])
      end if curr_array
    end
    return result
  end

  def self.get_currency_values(curr_array, array_of_hash)
    values = Hash.new()
    array_of_hash.each do |m|
      if curr_array.include?(m[:CharCode])
        values[m[:CharCode]] =  format_currency(m[:Value])
      end
    end
    return values
  end


  def self.format_currency(value)
    ((value.to_s.gsub(',','.').to_f)*100).to_i / 100.0
  end

  def self.weather(city=34560)
    xml_data  = Hpricot.XML(self.get_data(:base_uri => "http://www.google.com/ig/api?weather=moscow,russia").body)
    temp_hash = Hash.new()
    data      = ((xml_data/'current_conditions')/'temp_c')[0]
    unless !data
      temp_hash[:temperature] = data.attributes['data']
      temp_hash[:image]       = 'http://google.com'+((xml_data/'current_conditions')/'icon')[0].attributes['data']
    end
    return temp_hash
  end


  def self.cars
    #                                              :city/:amount_of_ads
    parse("http://sale.cars.ru/service/get_avto_xml/4/5/", [:id, :title, :rur, :photo, :descr])
  end

  def self.realty
    #                                                :city/:amount_of_ads
    parse("http://sale.cars.ru/service/get_realty_xml/4/4/", [:id, :title, :price])
  end

  def self.videos
    # parse ("http://video.cars.ru/service/get_xml/13/%E2%EE%EB%E3%EE%E3%F0%E0%E4/", [:id, :title, :screenshot])
    #    xml_data = Hpricot.XML(self.get_data(:base_uri => "http://video.cars.ru/service/get_xml/13/%E2%EE%EB%E3%EE%E3%F0%E0%E4/").body)
  end

  def self.competition
    url = 'http://foto.cars.ru/konkurses/get_by_region?region_id=4&mode=html'
    doc = Hpricot(self.get_data(:base_uri => url).body)
    return [] unless !doc.inner_text.blank?
    img =  doc.search('img')[0]['src'].gsub('\\','').gsub('"', '')
    link = doc.search('a')[0]['href'].gsub('\\','').gsub('"', '')
    text = doc.search('a')[0].search("a")[0].search("a")[0].inner_html.split('<\\/a>')[0].gsub("\\","%")
    return [text, link, img]
  end


  def self.news
    values =  parse("http://www.cars.ru/xml/news.xml", [:title, 'yandex:full-text'.to_sym, :link], :item)
    values.each do |hash|
      hash.each_pair { |key, value|  hash[key] = Iconv.iconv('UTF-8', 'WINDOWS-1251',  value)}
    end
    values.each do |news|
      id =  news[:link][0].scan(/\d+/)[0]
      image_url = "http://www.cars.ru/pictures/news/#{id}/picture-3.jpg"
      news[:image_url] = image_url
    end
    values
  end

  def self.parse(url, params_array, elements = :element)
    xml_data = Hpricot.XML(self.get_data(:base_uri => url).body)
    (xml_data/elements).reverse.collect do |element|
      temp_hash = Hash.new()
      puts (element).inspect
      params_array.each { |i| temp_hash[i] = (element/i).inner_html}
      temp_hash
    end
  end


  def self.remote_avatar(user_id)
    Hpricot(open("http://img.cars.ru/avatars/#{user_id}.jpg"))
  rescue OpenURI::HTTPError
    return false
  end

end
	# -------------------- Taking a peron name from E-mail header

	# === Returns Hash with firstname, lastname, and email from String:
	#
	# String "John Smith < john@acme.com >" will produce hash pairs:
	#
	# :firstname => "John",
	# :lastname => "Smith",
	# :email => "john@acme.com"
	#
	# String "< john@acme.com >" or "john@acme.com" will produce hash pairs:
	#
	# :firstname => "John",
	# :lastname => nil,
	# :email => "john@acme.com"
	def parse_mailfrom_field(from_field)
	sender = {}
	sender[:firstname] = from_field.scan(/((^)[<A-Za-z]+[^@\s])/).to_s.scan(/[A-Za-z]+/).to_s.capitalize
	sender[:lastname] = from_field.scan(/(\s[A-Za-z]+[^\s<])/).to_s[1..-1]
	sender[:email] = from_field.scan(/([^< \A]+@[^> $]+)/).to_s
	sender
	end

	# -------------------- Preparing an HTML e-mail to be shown on website
	def strip_fields
	deal = self.class.find(id)
	deal.from = deal.from.scan(/(([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,}))/i).flatten.first.downcase
	stripped_body = deal.body.split('<html>') #deleting everything before <html> tag
	deal.body = '<html>'+stripped_body[1] unless stripped_body[1].blank?
	deal.save
	end


	# -------------------- Parser class to get HTML data and parse it

	class Parser
	require 'hpricot'
	require 'typhoeus'
	require 'open-uri'
	include Typhoeus
	define_remote_method :get_data, :timeout => 1000, :cache_responses => 3600 # 1 hour cache

	def self.currency(day = Time.now(), curr_array =['USD', 'EUR'])
	values = parse("http://www.cbr.ru/scripts/XML_daily.asp?date_req=#{day.strftime("%d/%m/%Y")}", [:Value, :CharCode, :Name], 'Valute' )
	yesterday_values = parse("http://www.cbr.ru/scripts/XML_daily.asp?date_req=#{(day-1.day).strftime("%d/%m/%Y")}", [:Value, :CharCode, :Name], 'Valute' )
	unless values.blank? && yesterday_values.blank?
	curr_today = get_currency_values(curr_array, values)
	curr_yesterday = get_currency_values(curr_array, yesterday_values)
	result = Array.new()
	curr_array.each do \|curr\|
	result << curr_today[curr]
	result << format_currency(curr_today[curr] - curr_yesterday[curr])
	end if curr_array
	end
	return result
	end

	def self.get_currency_values(curr_array, array_of_hash)
	values = Hash.new()
	array_of_hash.each do \|m\|
	if curr_array.include?(m[:CharCode])
	values[m[:CharCode]] = format_currency(m[:Value])
	end
	end
	return values
	end


	def self.format_currency(value)
	((value.to_s.gsub(',','.').to_f)*100).to_i / 100.0
	end

	def self.weather(city=34560)
	xml_data = Hpricot.XML(self.get_data(:base_uri => "http://www.google.com/ig/api?weather=moscow,russia").body)
	temp_hash = Hash.new()
	data = ((xml_data/'current_conditions')/'temp_c')[0]
	unless !data
	temp_hash[:temperature] = data.attributes['data']
	temp_hash[:image] = 'http://google.com'+((xml_data/'current_conditions')/'icon')[0].attributes['data']
	end
	return temp_hash
	end


	def self.cars
	# :city/:amount_of_ads
	parse("http://sale.cars.ru/service/get_avto_xml/4/5/", [:id, :title, :rur, :photo, :descr])
	end

	def self.realty
	# :city/:amount_of_ads
	parse("http://sale.cars.ru/service/get_realty_xml/4/4/", [:id, :title, :price])
	end

	def self.videos
	# parse ("http://video.cars.ru/service/get_xml/13/%E2%EE%EB%E3%EE%E3%F0%E0%E4/", [:id, :title, :screenshot])
	# xml_data = Hpricot.XML(self.get_data(:base_uri => "http://video.cars.ru/service/get_xml/13/%E2%EE%EB%E3%EE%E3%F0%E0%E4/").body)
	end

	def self.competition
	url = 'http://foto.cars.ru/konkurses/get_by_region?region_id=4&mode=html'
	doc = Hpricot(self.get_data(:base_uri => url).body)
	return [] unless !doc.inner_text.blank?
	img = doc.search('img')[0]['src'].gsub('\\','').gsub('"', '')
	link = doc.search('a')[0]['href'].gsub('\\','').gsub('"', '')
	text = doc.search('a')[0].search("a")[0].search("a")[0].inner_html.split('<\\/a>')[0].gsub("\\","%")
	return [text, link, img]
	end


	def self.news
	values = parse("http://www.cars.ru/xml/news.xml", [:title, 'yandex:full-text'.to_sym, :link], :item)
	values.each do \|hash\|
	hash.each_pair { \|key, value\| hash[key] = Iconv.iconv('UTF-8', 'WINDOWS-1251', value)}
	end
	values.each do \|news\|
	id = news[:link][0].scan(/\d+/)[0]
	image_url = "http://www.cars.ru/pictures/news/#{id}/picture-3.jpg"
	news[:image_url] = image_url
	end
	values
	end

	def self.parse(url, params_array, elements = :element)
	xml_data = Hpricot.XML(self.get_data(:base_uri => url).body)
	(xml_data/elements).reverse.collect do \|element\|
	temp_hash = Hash.new()
	puts (element).inspect
	params_array.each { \|i\| temp_hash[i] = (element/i).inner_html}
	temp_hash
	end
	end


	def self.remote_avatar(user_id)
	Hpricot(open("http://img.cars.ru/avatars/#{user_id}.jpg"))
	rescue OpenURI::HTTPError
	return false
	end

	end