extracts.rb

require 'thread'
require 'logger'
require 'yaml'
require 'active_support/core_ext/string'
require 'pony'
require 'securerandom'

def grouping(configs)
  grouped_by_mis = configs.group_by{ |s, c| c["script"] }
  mis1_groups = grouped_by_mis["mis1"].group_by{ |s, c| c["args"][3] }
  groups = grouped_by_mis.merge(mis1_groups)
  groups.delete("mis1")
  groups.map do |g, cs|
    q = Queue.new
    cs.each{ |c| q.push(c) }
    q
  end
end

Dir[File.expand_path('../*_exporter.rb', __FILE__)].each { |f| require_relative f }

extracts_config = YAML.load(File.read(File.expand_path("../extracts_config.yaml", __FILE__)))
groupings = grouping(extracts_config)

consumers = []
@log_str = ["XML EXTRACT: #{ DateTime.now.strftime "%d/%m/%Y %H:%M" }"]
@log_messages = Hash.new{ |h, k| h[k] = [] }
groupings.each_with_index do |group, worker|
  consumers << Thread.new do
    log_file = "#{DateTime.now.strftime("%Y%m%d%H%M%S")}_log_#{worker}.txt"
    log = Logger.new("/home/company/extract_logs/#{log_file}")
    while (school_name, config = group.pop(true) rescue nil) do
      time = Time.now
      log.info "Processing File for #{school_name}"
      puts "Worker #{worker} running #{school_name}'s data extract (#{config['script']}). Queue size: #{1 + group.length}"
      failed = false

      begin
        "#{config['script']}_exporter".classify.constantize.new(config['args'], log).run!
      rescue Exception => e
        error_code = SecureRandom.hex(4)
        log.error "ERROR #{error_code}: #{school_name}: #{e.to_s}\n#{e.backtrace}"
        @log_messages[config['script']] << "#{DateTime.now.strftime "%d/%m/%Y %H:%M"} ***FAILED*** #{school_name}. ERROR: #{e.to_s}, #{log_file}: #{error_code}"
        failed = true
      end
      puts "Extract for #{school_name} finished in #{(Time.now - time).round(2)}s"

      unless failed
        @log_messages[config['script']] << "#{DateTime.now.strftime "%d/%m/%Y %H:%M"} Extract completed for #{school_name} in #{(Time.now - time).round(2)}s"
      end
    end
  end
end

consumers.each { |c| c.join }