URL_parsing_stuff.rb

require 'csv'
require 'net/http'
obsolete_metros = []

class OldMetro
  attr_accessor :url, :state
  def initialize(url, state)
    @url = url
    @state = state
  end
  
  def to_s
    "#{@state}/#{@url}"
  end
  def urls
    urls = []
    ['Lawn-Care', 'Outdoor-Pest-Control', 'Tree-Shrub-Care'].each do |service|
      urls << "/#{service}/#{@state}/#{@url}"
    end
    urls
  end
end

CSV.foreach('FuegoMetros.csv') do |row|
  obsolete_metros << OldMetro.new(row[1], row[4]) if row[2].nil? || row[2] == ''
end

sem_urls = []
File.open('semurls.txt', 'r+') do |file|
  file.each_line { |line| line.gsub(/\?/, '').strip!; sem_urls << line unless sem_urls.include?(line) }
end

sem_urls.each do |url|
  is_obsolete = false
  obsolete_metros.each do |obs|
    if obs.urls.include?(url)
      is_obsolete = true
      break
    end
  end
  unless is_obsolete
    status = Net::HTTP.get_response(URI.parse(url)).inspect
    if status.index('404')
      puts url
    end
  end
end

# File.open('redirects.csv', 'w+') do |file|
#   file.write("URLName,RedirectURL,StatusCode\n")
#   ['Lawn-Care', 'Outdoor-Pest-Control', 'Tree-Shrub-Care'].each do |service|
#     obsolete_metros.each do |obs|
#       file.write("/#{service}/#{obs.state}/#{obs.url},/#{service}/#{obs.state},302\n")
#     end
#   end
# end