ips dump

require 'ipaddr'

parameter :today do
  2.day.ago.to_date
end

helper :new_ips do
  NOP(:RDW).from(:"logs__#{today.ymd}").select_map { distinct(:ip) }
end

execute do

  new_ips.each_slice(500) do |slice| # insert in batches of 500 records

    # collect records to insert
    ips = slice.map do |ip|
      response = Nop.io(:GDS).get("#{ip}") rescue puts("Failed for #{ip}")
      next unless response

      ipinfo   = response['ipinfo']
      network  = ipinfo['Network'] || {}
      ip       = IPAddr.new(ipinfo['ip_address'], Socket::AF_INET).to_i

      # build the record to insert
      [
        ip,
        ipinfo['Location'].try(:[], 'CityData').try(:[], 'postal_code'),
        network['connection_type'],
        network['line_speed'],
        network['OrganizationData'].try(:[], 'organization_type'),
        network['OrganizationData'].try(:[], 'home'),
        network['carrier'],
        today
      ]

    end.compact

    Nop.io(:ARC).loader(self).into('appnexus/#{today.ym}/ips_#{today.ymd}.txt.gz').load!(ips)

  end

end