Http bot example

gistfile1.rb

# Gets the first 10 results of the query without using google API
# And then prints them as a JSON
# Usage google.rb [your search]

require "net/https"
require "uri"
require "nokogiri"
require "json"

q = URI::encode(ARGV[0])

uri = URI.parse("https://google.com")
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
path = "/search?output=search&sclient=psy-ab&q=#{q}"

headers = {
'accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
# 'accept-encoding' => 'gzip,deflate,sdch',
'accept-language' => 'es-ES,es;q=0.8,en;q=0.6',
'cookie' => '',
'referer' => '',
'user-agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36',
'x-client-data' => ''
}

response = http.get path, headers

#Follow the redirect 
if response.code == "301" || response.code == "302"  
	response = http.get response['Location']
end

document = Nokogiri::HTML(response.body)

results_list = []

# Force Encoding! force_encoding("ISO-8859-1").encode("UTF-8")
document.css('li.g').each do |result|   
	results_list.push({
	 	:title => result.css('h3.r').css('a').text.force_encoding("ISO-8859-1").encode("UTF-8"), 
	 	:link => result.css('h3.r').css('a').first["href"].force_encoding("ISO-8859-1").encode("UTF-8"),
	 	:content => result.css('span.st').text.force_encoding("ISO-8859-1").encode("UTF-8")
	 })
end

puts results_list.to_json