uzbek.rb

  desc "Normalize Uzbek file."
  task :uzbek => :environment do
    require 'csv'

    counter = 0
    entry_counter = 0
    previous = nil
    desc_array = []
    new_file = File.open("/Users/woodchip/fixed_UE.txt", 'w')

    CSV.foreach("/Users/woodchip/UE.txt", { :col_sep => "|" }) do |row|
      if row.count > 2
        row[1] = row[1] + row[2]
        row.pop
      end
      unless row.empty?
        entry = row[0].strip
        # p "ENTRY: #{entry}"
        pos = entry.match(/\w*\.$/).to_s.strip
        # p "PoS: #{pos}"
        entry = entry.gsub(/\w*\.$/, '').strip
        cyrillic = entry.gsub(/\w/u,'').strip
        # p "CYRILLIC: #{cyrillic}"
        latin = entry.gsub(/[Ѐ-ӿ\d]+\b/i,'').strip
        # p "LATIN: #{latin}"
        desc_array << ["<i>#{pos}</i>", row[1].strip]
        # p "DESC ARRAY: #{desc_array}"

        formatted_entry = "#{latin} / #{cyrillic}".gsub(/(^\!|\s\-|\-\s|\s\'|^\/|\/$|\'\s|\s\’|\’\s|\s\,|\,\s|^\?|\'$|\s\!|\s\?)|\([\ -\/]*\)/, '').strip
        # puts formatted_entry
        if previous.blank? || previous == entry
          previous = entry
          # p "#{previous} || #{entry}"
        else
          # p "#{previous} || #{entry}"
          new_entry = "<b>#{formatted_entry}</b>"
          hash = {}
          desc_array.each do |y|
            if hash[y[0]]
              hash[y[0]] = hash[y[0]] << ", #{y[1]}"
            else
              hash[y[0]] = y[1]
            end
          end
          hash.each_with_index do |(k, v), i|
            if i == 0
              new_entry << " #{k} #{v}"
            else
              new_entry << "; #{k} #{v}"
            end
          end
          # p new_entry
          new_file << new_entry + "\n"
          # p new_entry.encoding.name
          desc_array = []
          previous = entry
          entry_counter += 1
        end
        counter += 1
        if counter%1000 == 0
          p counter
        end
      end
    end
    new_file.close

    p "Done! ENTRIES COUNT: #{entry_counter}"
    p "Done! TOTAL COUNT: #{counter}"    
  end