source-customers.rb

require 'dbf'
require 'pry'
require_relative './dbfutil'
require_relative './Progressive'

require 'nested_multimap'


class SourceCustomers
   prepend Progressive

   def initialize(input_file = 'C:\\NetClipDev\\CLIPv\\MAINDBF\\customer.dbf', *options)
      @table ||= DBF::Table.new(input_file)
      @table.nmm_summary(SourceCustomers.masked_columns, SourceCustomers.nmm)
   end
   
   def self.count(handle)
      @table ||= DBF::Table.new(handle)
      @total = @table.record_count
   end


   def self.mapped_columns
      hash = {}
      hash['num'] = 'peachtree_id'
      hash['paddress'] = 'ADDRESS'
      #hash['pcity'] = 'CITY'		# needs regex to strip
      hash['pstate'] = 'STATE'
      hash['pzip'] = 'ZIP_CODE'
      hash['cu_website'] = 'website'
      hash
   end

   def self.nmm



      @last_first_last_first = /(?<last1>.+) , \s* (?<first1>.+) \s+ (?<conn>[&\/]) \s+ (?<last2>.+) , \s+ (?<first2>.+) \s+ (?<lsm>-\s+LSM)?/x
      @partners              = /(?<last>.+) , \s+ (?<first>.+) \s* (?<conn>[&]) \s+ (?<extra>[^\n\s]+)? (?<lsm>-\s+LSM)? $/x
      @last_first_opt_extra = /(?<last>.+) , \s+ (?<first>[^&()\\\/\s]+) \s* (?<extra>[&\()\\\/]*[^\n\s]+)? \s*? (?<lsm>-\s+LSM)? $/x
      @lsm_suffix = /(?<name>.*)\s+-\s+LSM\s*/x
      @all = /.*/

      ctransform ||= NestedMultimap.new
      ctransform['customer','num'] = ['contact','peachtree_id']
      ctransform['customer','paddress'] = ['contact','ADDRESS']
      ctransform['customer','pstate'] = ['contact', 'STATE']
      ctransform['customer','pzip'] = ['contact', 'ZIP_CODE']
      ctransform['customer','cu_website'] = ['contact', 'website']
      ctransform['customer','pcity'] = ['contact', 'CITY', Proc.new { |s| s.sub(/,\s*$/, '').upcase }]
   
      ctransform['customer','num'] = ['contact_billing','peachtree_id']
      ctransform['customer','baddress'] = ['contact_billing', 'address']
      ctransform['customer','bstate'] = ['contact_billing', 'state']
      ctransform['customer','bzip'] = ['contact_billing', 'zip_code']
   
      ctransform['customer','name', @last_first_last_first] = ['contact', 'COMPANY', ->(match) {"#{match.string}"}]
      ctransform['customer','name', @last_first_last_first] = ['contact', 'CONTACT', ->(match) {"#{match[:first1].upcase}"}]
      ctransform['customer','name', @last_first_last_first] = ['contact', 'last_name', ->(match) {"#{match[:last1].upcase}"}]

   
      ctransform['customer','name',@partners] = ['contact', 'COMPANY', ->(match)   { "#{match.string}" }]
      ctransform['customer','name',@partners] = ['contact', 'CONTACT', ->(match) { "#{match[:first].upcase}#{match[:conn]} #{match[:extra].upcase}" } ]
      ctransform['customer','name',@partners] = ['contact', 'last_name',-> (match) { "#{match[:last].upcase}" }]

      ctransform['customer','name',@last_first_opt_extra] = ['contact','COMPANY', ->(match) {"#{match.string}"}]
      ctransform['customer','name',@last_first_opt_extra] = ['contact','CONTACT', ->(match) {"#{match[:first].upcase}"} ]
      ctransform['customer','name',@last_first_opt_extra] = ['contact','CONTACT', ->(match) {"#{match[:first].upcase} #{match[:extra].upcase}" unless match[:extra].nil?}]
      ctransform['customer','name',@last_first_opt_extra] = ['contact','last_name',->(match) {"#{match[:last].upcase}"}]
   
      ctransform['customer','name',@all] = ['contact','COMPANY', ->(match) {"#{match.string}"} ]

      ctransform
   end

   def self.masked_columns
      list = []
      list << 'CU_ID'
      list << 'PHONE2'
      list << 'PHONE3'
      list << 'PHONE4'
      list << 'CU_EM1SEND'
      list << 'CU_EM2SEND'
      list << 'CU_EM3SEND'
      list << 'CU_EM4SEND'
      list << 'CU_EMAIL4'
      list << 'CU_NOPRST'
      list << 'CU_EMDESC'
      list << 'CU_EMDESC1'
      list << 'CU_EMDESC2'
      list << 'CU_EMDESC3'
      list << 'CU_EMDESC4'
      list << 'QB_ITEM'
      list << 'QB_CLASS'
      list << 'CU_PRE_INS'
      list << 'QCREDIT'
      list << 'QDEBIT'
      list << 'DOCUMENT'
      list << 'CU_CREW'
      list << 'CU_MAP'
      list << 'CU_ROUTE'
      list << 'CU_CONT'
      list << 'CU_LSQ'
      list << 'CU_DIFF'
      list << 'CU_OCOPY'
      list << 'CU_RNOTE'
      list << 'CU_TAX'
      list << 'CU_ZONE'
      list << 'CU_MAP_LON'
      list << 'CU_MAP_LAT'
      list << 'CU_MAP_USE'
      list << 'CU_SVCGRP'
      list << 'CU_RTNOTES'
      list << 'CU_PICT1'
      list << 'CU_PDESC1'
      list << 'CU_ATTACH1'
      list << 'CU_ATTD1'
      list << 'CU_PICT2'
      list << 'CU_PDESC2'
      list << 'CU_ATTACH2'
      list << 'CU_ATTD2'
      list << 'CU_PICT3'
      list << 'CU_PDESC3'
      list << 'CU_ATTACH3'
      list << 'CU_ATTD3'
      list << 'CU_PICT4'
      list << 'CU_PDESC4'
      list << 'CU_ATTACH4'
      list << 'CU_ATTD4'
      list << 'UDF_N2'
      list << 'CU_PWORD'
      list << 'CU_CHGEOM'
      list << 'CU_CHGDOM'
      list << 'CU_CHRGDAT'
      list << 'CU_CHGALJB'
      list << 'CU_CHGINST'
      list << 'QB_TERMS'
      list << 'QB_ID'
      list << 'QB_EMP_ID'
      list << 'CU_SOURCE'
      list << 'CU_ESTDATE'
      list << 'INST_NOTES'
      list << 'LISTID'
      list << 'CU_BPRRNO1'
      list << 'CU_BPRRNO2'
      list << 'CU_BPRRNO3'
      list << 'CU_BPRRNO4'
      list
   end

   def each
     @table.each do |row|
         progress!
         yield(row.attributes_ignoring(SourceCustomers.masked_columns.to_a)) unless row.nil?
     end
     @table.close
     stop!
   end 
end

#################
require File.expand_path('../config/environment', File.dirname(__FILE__))



class DestinationContacts
   def initialize(count, options={})
       Rails.application.eager_load!
       @c = Contact.connection

       @map ||= {}
       @map = options[:map]
       @nmm = options[:nmm]

       @pb = options[:progresses_with]
   end

   def monitor(a = [])
	@watched ||= []
	@watched.push(a)
        @watched.flatten
   end

   def write(row)
      return if (row[:num] == -1)

      Contact.transaction do
         contact = Contact.where(:peachtree_id => row[:num]).first_or_initialize()
   	 
         # the id
         monitor(:peachtree_id)

         if contact.new_record?
          #@pb.log "CREATE"
         else
          #@pb.log "UPDATE"
         end
         
         @nmm.chunk { |k| [k.first.first, k.first.second] }.each { |chunk, targets|

               from_table_column = chunk
               from_table = chunk.first
               from_column = chunk.second.to_sym
               
               failed_regexes = []
               good_regex = nil

               targets.each { |x|
                   from_regex = x.first.third
                   to_table = x.second.first
                   to_column = x.second.second
                   to_proc = x.second.third

                   next if failed_regexes.include? from_regex
                   next unless to_table == 'contact'
                   
                   case

                   when (from_regex.is_a?(Regexp) && to_proc.is_a?(Proc) && !(failed_regexes.include? from_regex) && (from_regex == good_regex || good_regex == nil) )
                      if (match = from_regex.match(row[from_column]))
                               contact[to_column] = @c.quote_string to_proc.call(match) unless to_proc.call(match).nil?
                               good_regex = from_regex # make sure we don't match any new regexes after this
                      else 
                         failed_regexes << from_regex
                         next
                      end
                   when ((not from_regex.is_a?(Regexp)) && to_proc.is_a?(Proc))
                      contact[to_column] = @c.quote_string to_proc.call(row[from_column])
                   when ((not from_regex.is_a?(Regexp)) && (not to_proc.is_a?(Proc)))
                      contact[to_column] = @c.quote_string row[from_column] if row[from_column]
                   else
                      next
                   end
                   @pb.log '%-20s => %-20s := %-30s' % ["#{from_column}","#{to_table}.#{to_column}",contact[to_column]] unless false
                   monitor(to_column)
               }
            }
            @pb.log ''

      end #of transaction
   end

   def close
   end
end
#####################


#!/usr/bin/env ruby

require 'pry'

require_relative 'SourceCustomers'
require_relative 'DestinationContacts'
require_relative 'Transformations'


source_file = 'C:\\NetClipDev\\CLIPv\\MAINDBF\\customer.dbf'

@pb = ProgressBar.create( 	title: "#{self.class.name}",
					total: SourceCustomers.count(source_file),
					throttle_rate: 0.10,
					autofinish: false
					)



pre_process do
   @pb.log '[PRE]'
end


source SourceCustomers, source_file, progresses_with: @pb, total: SourceCustomers.count(source_file)


#transform SkipBadCustomers, ''


destination DestinationContacts, '',  progresses_with: @pb, map: SourceCustomers.mapped_columns, nmm: SourceCustomers.nmm


post_process do
   @pb.log '[POST]'
end