worker.rb

class ContactImportWorker
  include Sidekiq::Worker

  require 'sidekiq'
  require 'csv'
  # require 'sidekiq/testing/inline'

  sidekiq_options queue: :med
  #sidekiq_options retry: false

  def perform(file_id, column_hash, campaigns)
    @columns = process_columns(column_hash)
    @campaigns = campaigns
    
    file_to_process = UploadedFile.find(file_id)
    user = User.find(file_to_process.user_id)
    begin
      # ActiveRecord::Base.transaction do

        case File.extname(file_to_process.file_file_name).downcase
          when '.csv' then
            process_csv(file_to_process, user, 'UTF-8')
          when '.xls' then
            process_xls(file_to_process, user)
          when '.xlsx' then
            process_xlsx(file_to_process, user)
          else
            raise "Unknown file type: #{file_to_process.original_filename}"
        end

        file_to_process.processed = true
        file_to_process.save
      # end
    rescue => e
      raise_error(file_to_process, e)
    end

  end

  def raise_error(file_to_process, error)
    file_to_process.error = error.to_s
    file_to_process.processed = false
    file_to_process.save
    FileUploadMailer.email_admin(User.find(file_to_process.user_id), file_to_process, error).deliver
    raise error
  end

  def process_xls(file_to_process, user)
    file_path = File.dirname(file_to_process.file.path)
    file_basename = File.basename(file_to_process.file_file_name, ".xls")

    filename = Rails.root.join(file_path, file_basename + ".csv")

    xls = Roo::Spreadsheet.open(file_to_process.file.path, extension: :xls)
    xls.to_csv(filename)

    if File.exists? filename
      file = File.open(filename)
      file_to_process.file = file
      file_to_process.save
      file.close
    end

    process_csv(file_to_process, user, 'ISO-8859-1')
  end

  def process_xlsx(file_to_process, user)
    file_path = File.dirname(file_to_process.file.path)
    file_basename = File.basename(file_to_process.file_file_name, ".xlsx")

    filename = Rails.root.join(file_path, file_basename + ".csv")

    xlsx = Roo::Spreadsheet.open(file_to_process.file.path, extension: :xlsx)
    xlsx.to_csv(filename)

    if File.exists? filename
      file = File.open(filename)
      file_to_process.file = file
      file_to_process.save
      file.close
    end

    process_csv(file_to_process, user, 'ISO-8859-1')
  end


  # 33/sec
  # without existing check is 4 seconds faster
  def process_csv(file_to_process, user, encoding='')
    clear_file file_to_process.file.path
    keys = {
      first_name: [:name, :first, :first_name, :title], 
      last_name: [:last_name, :last, :surname, :contact], 
      number: [:phone, :number, :phone_number, :phone1, :phone_1, :phonenumber, :phone_number_combined], 
      company: [:company], 
      notes: [:notes, :comments]
      }
      array = []
    CSV.foreach(file_to_process.file.path, { headers: true, return_headers: false}) do |row|
      row_hash = process_row(row)
      array << row_hash
      process_contact(row_hash, user, keys)
    end
  end

  def clear_file(file_name)
    text = File.read(file_name)
    formated = text.gsub(/\0/, '').gsub("\u0000", '').chomp
    encoding_options = {
      :invalid           => :replace,  # Replace invalid byte sequences
      :undef             => :replace,  # Replace anything not defined in ASCII
      :replace           => '',        # Use a blank for those replacements
      :universal_newline => true       # Always break lines with \n
    }
    encoded = formated.encode(Encoding.find('ASCII'), encoding_options)
    File.open(file_name, "w") {|file| file.puts encoded }
  end

   def process_row(row)
    hash = {}.with_indifferent_access
    @columns.each do |k,v|
      hash[k] = row.values_at(v).first
    end
    hash
   end

  def process_columns(hash)
    hash.delete_if { |k, v| v.empty? }
    columns = {}.with_indifferent_access
    hash.invert.each{|k,v| columns[k.parameterize('_').to_sym] = v.to_i}
    columns
  end

  def process_contact(contact_hash, user, keys)
    contact = Contact.new(contact_hash)
    contact.user_id = user.id
    if contact.save
      if @campaigns.present?
        @campaigns.split(',').each do |camp|
          campaign = Campaign.find_or_create_by(:user_id => user.id, name: camp)
          campaign.contacts << contact
        end
      end
    else
      puts "=================================ERROR!!!!    " + contact.errors.full_messages.to_s
    end
  end

end