Tab Delimited File Import

common.rb

  def import_file(conn, file_path, schema, table_name, rules, table_columns = nil, take_how_many = nil)
    begin
      logger.info("Running COPY command with data for #{file_path} ...")

      table_columns_string = table_columns.nil? ? '' : "(#{table_columns})"
      conn.exec( "COPY #{schema}.#{table_name}#{table_columns_string} FROM #{rules}" )

      file = convert_file_to_utf8(file_path)
      file.each_line do |line|
        column_data = line.gsub(/\r/," ").gsub(/\n/," ").split("\t")
        data = take_how_many.nil? ? line : "#{column_data.take(take_how_many).join("\t")}\n"

        ### Uncomment this to test error-handling for exceptions from the reader side:
        # raise Errno::ECONNRESET, "socket closed while reading"
        # logger.info("MaxmindImporter - sending %d bytes of data..." % [ line.length ])
        until conn.put_copy_data( data.force_encoding('UTF-8') )
          logger.info("waiting for connection to be writable...")
          sleep 0.1
        end
      end

      conn.put_copy_end
      while res = conn.get_result
        unless res.result_status == 1
          logger.fatal("#{res.res_status(res.result_status)} - #{res.result_error_message}")

          raise "Parsing Issue: #{res.result_error_message}"
        end

        logger.info("Result of COPY is: %s" % [ res.res_status(res.result_status) ])
      end
    rescue Errno => err
      errmsg = "%s while reading copy data: %s" % [ err.class.name, err.message ]
      conn.put_copy_end( errmsg )

      while res = conn.get_result
        logger.error("result_error_message: #{res.result_error_message}") unless res.result_error_message.nil? || res.result_error_message.empty?
      end

      logger.fatal("#{errmsg}")

      raise err
    end
  end