Instantly share code, notes, and snippets.

Embed
What would you like to do?
Rails Copybara scrap example
require 'rails_helper'
require 'rspec/rails'
feature "scrapper" do
it "gets all companies which export to austria" do
# first lets visit setting where we will include financial data inside our search results
visit "http://www.sloexport.si/default.asp?MenuID=279&Settings=true"
sleep(0.3)
find(:xpath, ".//input[@value='8']").click
find(:xpath, ".//input[@name='btnResultSettings']").click
country = "Austria"
visit "http://www.sloexport.si/default.asp?MenuID=279&advancedSearch=true"
find_by_id('dialog-drzave-link').click
sleep(0.3)
find(:xpath, ".//input[@desc='" + country + "']").click
find_by_id('btnChoose').click
find(:css, "input.button_adv_search").click
# go trough every page
url = current_url
while true do
page.all(:css, '.company').each do |el|
company = Company.new
company.name = el.all(:css, ".CompName").first.text
company.address = el.find(:css, ".address").text
objects = el.all(:css, "b")
if objects.count > 0
income = el.all(:css, "b").first.text
if income.length > 2
company.income = income.gsub(',', '').split(".").first.to_i
else
company.income = -1;
end
else
company.income = -1;
end
objs = el.all(:css, ".contacts")
if objs.count > 0
company.full_contacts = objs.first.text
emails = objs.first.all(:css, "a")
if emails.count > 0
company.email = emails.first.text
end
end
company.country = country
company.save!
puts company.id.to_s + ": " + company.name
company.reload
end
if have_css('a.searchnext')
page.all(:css, "a.searchnext").first.click
if url == current_url # it seems like we reached the last page
break
end
url = current_url
else
break
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment