Full advantage threads handle with JRuby - Sidekiq - Sinatra - Smart_csv

_progress_job_view.html.erb

<div class="modal-content center-text">
  <h4 style="padding: 20px;line-height: initial;"> 
    The file is processing, please be patient, Thanks! 
  </h4>
  <p style="margin-bottom:30px;"> <%= image_tag("spinner.gif", size: "100") %> </p>
  
    <% if !import.finished? %>
      <h3>
        It's been processed
        <%= import.row.nil? ? 0 : (import.row.to_i - import.total_rows).abs %>
        rows from
        <%= import.total_rows %>
      </h3>
    <% end %>
  
  <h5 style="padding:10px;">
    CRM Team!
  </h5>
  <p> Time of execution: <%= distance_of_time_in_words(import.starter_at, Time.now) unless import.starter_at.nil? %> | <%= (import.row.to_i - import.total_rows).abs %>
  </p>
</div>

Import.rb

# APP 1: Import Class

class Import
  include Mongoid::Document
  include Mongoid::Timestamps
  include Mongoid::Paperclip

  field :starter_at, type: Time
  field :finished_at, type: Time
  field :total_rows, type: Integer
  field :row, type: String
  field :error, type: Boolean

  def finished?
    finished_at.present?
  end

  def failure?
    error?
  end
  
end

imports_controller.rb

# APP 1: Rails 4.1, Ruby 2.1.0
# Controller where import excel file and post it id and path to APP 2 to start processign.
...

def create
    @import = Import.new(import_params)
    if @import.save
      @import.update_attributes(starter_at: Time.now)
      
      # Here make a post request to APP 2
      Typhoeus.post(
        "http://address_to_app2/posts", 
        body: {import_id: @import.id, file_path: @import.file.path}
      )
        
      flash[:success] = "load perfect!"
    else
      flash[:error] = []
      @import.errors.full_messages.each {|msg| flash[:error] << msg }
    end
  else
    flash[:error] = "Choose a file"
    render :new
  end
end

def progress_job
  import = Import.find(import_id)
  respond_to do |format|
    format.json  { render :json => import.finished? }
  end
end

def progress_job_view
  @import = Import.find(import_id)
  unless @import.finished?
    Typhoeus.get("http://address_to_app2/progress", body: {import_id: @import.id})
  end
end

...

show.html.erb

<%= render :partial => "#{path_to/progress_job_view}" locals: {import_id: import_id} %>

<script type="text/javascript">
  
  $.smartPoller = function(wait, poller) {
    if ($.isFunction(wait)) {
      poller = wait
      wait = 1000
    }
 
    (function startPoller() {
      setTimeout(function() {
        poller.call(this, startPoller)
      }, wait)
      wait = wait * 2 // Multiplier set interval for polling request import file status
    })()
  }
  var import_id = $("#import_id").val();
  if (import_id){
    $.smartPoller(function(retry) {
      // It's finish?
      $.getJSON('/imports/'+import_id+'/progress_job', function(finished) {
        if (finished == false) {
          $.get('/imports/'+import_id+'/progress_job_view/', null, null, "script");
          retry();
        }else{
          $.get('/imports/'+import_id+'/progress_job_view/', null, null, "script");
        }
      });
    });
  }
</script>

Worker.rb

# APP 2: JRuby, Sidekiq, Sinatra
# Read from excel a little bit more than 10,000 lines and processed like charm.
# Including JS Backend polling request to get status of the processes files from other app

class ExcelWorkerServer < Sinatra::Base

  require 'sinatra'
  require 'sidekiq/api'
  require 'smarter_csv'
  require_relative 'worker'

  post '/import/:id/file/:path' do
    imported = Import.find(params[:id])
    csv = SmarterCSV.process(params[:path], {:strings_as_keys => true, :remove_unmapped_keys => true, :col_sep => "\t"})
    imported.update_attributes(total_rows: csv.size)
    csv.each do |row|
      # Start Sidekiq to process every row
      ExcelWorker.perform_async(row, params[:id])
    end
  end

  # Aks if the import file finish it processing
  # through 'excel' queue size
  # once it is cero(0) then its finished without erros
  get '/progress/:import_id/status' do
    import = Import.find(params[:import_id])
    count_queue = Sidekiq::Queue.new("excel").size.to_s
    if count_queue == "0"
      import.update_attributes(finished_at: Time.now, row: count_queue)
    end
    import.update_attributes(row: count_queue)
    Sidekiq::Queue.new("excel").size.to_s
  end

end

class ExcelWorker
  include Sidekiq::Worker
  sidekiq_options queue: 'excel' # name of the queue where sidekiq have every thread to process all readed rows
  
  def perform(row, import_id)
    row.each do |key, value|
      data = {name: key, value: value, import_id: import_id}
      ---
      # Save each row into DB, the best way you desire...
      ---
    end
  end
end