Parse Ciber Records

parse-ciber.rb

#!/usr/bin/env ruby

# Author: Trevor Strieber
# Date  : Nov 08 2012

require 'zlib'
require 'date'

def parse_line(line)
  {
    record_type:                        line[0..1],
    return_code:                        line[2],
    ciber_record_return_reason_code:    line[3..4],
    invalid_field_identifier:           line[5..7],
    home_carrier_sid:                   line[8..12],

    msid_indicator:                     line[13],
    msid:                               line[14..28],
    msisdn_mdn_length:                  line[29..30],
    msisdn_mdn:                         parse_field_with_length(line[29..30], line[31..45]),
    esn_uimid_imei_meid_indicator:      line[46],
    esn_uimid_imei_meid:                line[47..65],

    serving_carrier_sid:                line[66..70],
    total_charges_and_taxes:            line[71..80],
    system_reserved_filler:             line[81],
    total_state_province_taxes:         line[82..91],
    system_reserved_filler2:            line[92],

    local_other_taxes:                  line[93..102],
    system_reserved_filler3:            line[103],
    call_date:                          parse_date_time(line[104..109], line[212..217]),
    call_direction:                     line[110],
    call_completion_indicator:          line[111],

    call_termination_indicator:         line[112],
    caller_id_length:                   line[113..114],
    caller_id:                          parse_field_with_length(line[113..114], line[115..129]),
    called_number_length:               line[130..131],
    called_number_digits:               parse_field_with_length(line[130..131], line[132..146]),

    location_routing_number_length:     line[147..148],
    location_routing_number:            parse_field_with_length(line[147..148], line[149..163]),
    tldn_length:                        line[164..165],
    tldn:                               parse_field_with_length(line[164..165], line[166..180]),
    currency_type:                      line[181..182],
    system_reserved_filler4:            line[183..184],

    original_batch_sequence_number:     line[185..187],
    initial_cell_site:                  line[188..198],
    time_zone_indicator:                line[199..200],
    daylight_savings_indicator:         line[201],
    message_accounting_digits:          line[202..211],

    air_connect_time:                   parse_date_time(line[104..109], line[323..328]),
    air_chargeable_time:                parse_duration(line[218..223]),
    air_elapsed_time:                   parse_duration(line[224..229]),
    air_rate_period:                    line[230..231],
    air_multi_rate_period:              line[232],
    air_charge:                         line[233..242],

    system_reserved_filler5:            line[243],
    other_charge1_indicator:            line[244..245],
    other_charge1:                      line[246..255],
    system_reserved_filler6:            line[256],
    system_reserved_filler7:            line[257..269],

    printed_call:                       line[270..284],
    fraud_indicator:                    line[285..286],
    fraud_sub_indicator:                line[287],
    special_features_used:              line[288..292],
    called_place:                       line[293..302],
    called_state_province:              line[303..304],

    called_country:                     line[305..307],
    serving_place:                      line[308..317],
    serving_state_province:             line[318..319],
    serving_country:                    line[320..322],
    toll_connect_time:                  line[323..328],
    toll_chargeable_time:               parse_duration(line[330..334]),

    toll_elapsed_time:                  parse_duration(line[335..340]),
    toll_tariff_descriptor:             line[341..342],
    toll_rate_period:                   line[343..344],
    toll_multi_rate_period:             line[345],
    toll_rate_class:                    line[346],
    toll_rating_point_length_indicator: line[347..348],

    toll_rating_point:                  parse_field_with_length(line[347..348], line[349..358]),
    toll_charge:                        line[359..368],
    system_reserved_filler8:            line[369],
    toll_state_province_taxes:          line[370..379],
    system_reserved_filler9:            line[380],
    toll_local_taxes:                   line[381..390],

    system_reserved_filler10:           line[391],
    toll_network_carrier_id:            line[392..396],
    local_carrier_reserved:             line[397..471],
    system_reserved_filler11:           line[472..546]
  }
end

def get_header(record)
  record.to_a.transpose[0].join(" | ") << "\n"
end

def convert_record(record)
  record.to_a.transpose[1].map{ |field| field.to_s.strip}.join(" | ") << "\n"
end

def parse_date_time(date, time)
  valid_date_time(date, time) ? DateTime.parse("20" + date + time).strftime("%Y%m%d%H%M%S") : ""
end

def valid_date_time(date, time)
  !date.nil? && !time.nil? && date != "" && time != "" && date != "000000" && time != "000000"
end

def parse_duration(duration)
  valid_duration(duration) ? duration[0..3].to_i * 60 + duration[4..5].to_i : ""
end

def valid_duration(duration)
  !duration.nil? && duration != ""
end

def parse_field_with_length(length, field)
  valid_field(field) ? field[0..length.to_i - 1] : ""
end

def valid_field(field)
  !field.nil? && field != ""
end

abort('usage: parse-ciber.rb input-dir output-dir') unless ARGV.size == 2
input, output  = ARGV[0..1]
abort("directories must exist!") unless File.directory?(input) && File.directory?(output)

Dir.chdir(input)
files = Dir.glob("*.gz")
files.each do |file_name|
  File.open(file_name) do |input_file|
    puts "#{input}/#{file_name} -> #{output}/#{file_name}-parsed"
    File.open("../#{output}/#{file_name}-parsed", 'w') do |output_file|
      output_file.write(get_header(parse_line("")))
      gz = Zlib::GzipReader.new(input_file)
      gz.each_line do |line|
        output_file.write(convert_record(parse_line(line))) if line[0..1] == "22"
      end
    end
  end
end