Skip to content

Instantly share code, notes, and snippets.

@Azdaroth
Created November 27, 2016 13:46
Show Gist options
  • Save Azdaroth/17ee688d9b1098f5bfa8bfeeed0c4479 to your computer and use it in GitHub Desktop.
Save Azdaroth/17ee688d9b1098f5bfa8bfeeed0c4479 to your computer and use it in GitHub Desktop.
class NevacheImport
attr_reader :account, :base_url, :mechanize, :csv_dir, :rentals_csv_file,
:rentals_details_csv_file, :addresses_csv_file
def initialize(account:, csv_dir:)
@account = account
@base_url = "http://www.nevache-tourisme.fr/fr/locations-meubles/"
@mechanize = Mechanize.new
@csv_dir = csv_dir
@rentals_csv_file = File.join(csv_dir, "a_structures.csv")
@rentals_details_csv_file = File.join(csv_dir, "a_structures_detail.csv")
@addresses_csv_file = File.join(csv_dir, "a_adresses.csv")
end
def import!
Importer.new.import!(account, full_mapping)
end
def main_page
@main_page ||= mechanize.get(base_url)
end
def rentals_urls
@rentals_urls ||= begin
main_page.links_with(class: "structure-resume").map(&:href).map do |relative_url|
"#{base_url}#{relative_url}"
end
end
end
def rentals_mappings_from_page
@rentals_mappings_from_page ||= begin
rentals_urls.map do |url|
RentalPage.new(mechanize, url).rental_mapping
end
end
end
def full_mapping
rentals_mappings_from_page.map do |mapping|
mapping[:rental_attributes] = attributes_for_rental_from_mapping(mapping)
mapping[:rentals_contact_attributes] = attributes_for_rental_contact_from_mapping(mapping)
mapping[:rentals_amenities_keys] = rentals_amenities_keys(mapping)
mapping
end
end
def attributes_for_rental_from_mapping(mapping)
csv_row = rentals_from_csv.find { |r| r[:id].to_s == mapping[:id].to_s } || {}
details = rentals_details_from_csv.find { |r| r[:id_structure].to_s == csv_row[:id].to_s } || {}
address = addresses_from_csv.find { |r| r[:id].to_s == csv_row[:id_adresse].to_s } || {}
attributes = {}
attributes[:name] = csv_row[:libelle]
attributes[:zip] = csv_row[:cp]
attributes[:city] = csv_row[:commune]
attributes[:address1] = address[:libelle]
attributes[:description_fr] = strip(csv_row[:description])
attributes[:summary_fr] = strip(csv_row[:resume]).truncate(150)
attributes[:surface] = (details[:surface].presence && details[:surface].to_i)
attributes[:notes] = strip(details[:commentaire])
attributes[:surface_unit] = "metric" if attributes[:surface].present?
attributes[:bedrooms_count] = details[:nb_chambre].to_i
attributes[:currency] = "EUR"
attributes[:country_code] = "FR"
attributes
end
def attributes_for_rental_contact_from_mapping(mapping)
csv_row = rentals_from_csv.find { |r| r[:id].to_s == mapping[:id].to_s } or return {}
attributes = {}
attributes[:fullname] = csv_row[:email]
attributes[:email] = csv_row[:email]
attributes[:phones] = [csv_row[:tel], csv_row[:mobile]].compact.join(", ")
attributes[:spoken_languages] = ["FR"]
attributes
end
def rentals_amenities_keys(mapping)
details_amenities_mapping = {
animal: "pets_considered",
balcon_terrasse: "balcony",
jardin: "garden",
telephone: "telephone",
sauna_jacuzzi_bain: "jacuzzi",
acces_handicape: "handicap_accessible",
acces_voiture: "car_included",
cuisine: "full_equipped_kitchen",
parking: "parking_included",
tele: "tv",
wifi: "internet",
lave_linge: "washing_machine",
lave_vaisselle: "dishwasher"
}
keys = []
details = rentals_details_from_csv.find { |r| r[:id_structure].to_s == mapping[:id].to_s } || {}
details_amenities_mapping.each do |key_from_detail, key|
keys << key if details[key_from_detail].to_i == 1
end
keys
end
def rentals_from_csv
@rentals_from_csv ||= CSV.foreach(rentals_csv_file, csv_options).to_a
end
def rentals_details_from_csv
@rentals_details_from_csv ||= CSV.foreach(rentals_details_csv_file, csv_options).to_a
end
def addresses_from_csv
@addresses_from_csv ||= CSV.foreach(addresses_csv_file, csv_options).to_a
end
def csv_options
{
headers: true,
header_converters: :symbol
}
end
def strip(text)
ActionView::Base.full_sanitizer.sanitize(text.to_s, tags: [])
end
class RentalPage
attr_reader :rental_page, :url
def initialize(mechanize, url)
@rental_page = mechanize.get(url)
@url = url
end
def rental_mapping
{
url: url,
id: id,
photos: photos_urls,
}
end
def photos_urls
rental_page.search(".structure-photo > a").map { |el| el["href"] }
end
# http://www.nevache-tourisme.fr/fr/locations-meubles/la-moulini-re-i_47.htm
#0. http://www.nevache-tourisme.fr/fr/locations-meubles/la-moulini-re-i_47.htm
#1. http://www.nevache-tourisme.fr/fr/locations-meubles/la-moulini-re-i
#2. _
#3. 47
#4. .htm
def id
/(.+)(_)(\d+)(.+)/.match(url)[3]
end
end
class Importer
def import!(account, mapping)
ActiveRecord::Base.transaction do
mapping.each do |rental_mapping|
rentals_contact = account.rentals_contacts.where(rental_mapping[:rentals_contact_attributes]).first_or_create
if rentals_contact.valid?
rental_mapping[:rental_attributes].merge!(contact: rentals_contact)
end
rental = account.rentals.build(rental_mapping[:rental_attributes])
rental.save!
rental_mapping[:photos].each do |photo_url|
rental.photos.create!(remote_photo_url: photo_url)
end
rental_mapping[:rentals_amenities_keys].each do |key|
amenity = Amenity.find_by(key: key)
RentalsAmenity.create!(amenity: amenity, rental: rental)
end
end
end
end
end
end
class Importer
def import!(account, mapping)
ActiveRecord::Base.transaction do
mapping.each do |rental_mapping|
rentals_contact = account.rentals_contacts.where(rental_mapping[:rentals_contact_attributes]).first_or_create
if rentals_contact.valid?
rental_mapping[:rental_attributes].merge!(contact: rentals_contact)
end
rental = account.rentals.build(rental_mapping[:rental_attributes])
rental.save!
rental_mapping[:photos].each do |photo_url|
rental.photos.create!(remote_photo_url: photo_url)
end
rental_mapping[:rentals_amenities_keys].each do |key|
amenity = Amenity.find_by(key: key)
RentalsAmenity.create!(amenity: amenity, rental: rental)
end
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment