ED Scraper (ITS RUBY!)

gistfile1.rb

require 'mechanize'
require 'logger'
require 'io/console'
require 'json'


SETTINGS = ".ed_profile_scraper.yml"
COOKIES = ".ed_profile_scraper.cookies"

BASE_URL = "https://companion.orerve.net/"
LOGIN_URL = BASE_URL + "user/login"
CONFIRM_URL = BASE_URL + "user/confirm"
PROFILE_URL = BASE_URL + "profile"

def save_settings
  puts "Saving settings..."
  File.open(SETTINGS, "w") do |file|
    file.write $settings.to_yaml
  end
  puts "...done!"
end

begin
  $settings = YAML::load_file SETTINGS
rescue Errno::ENOENT
  puts "No settings found!"

  puts "Please type in your ED username (email): "
  email = gets.chomp

  puts "Please type in your ED password: "
  password = STDIN.noecho(&:gets).chomp
  
  $settings = { :email => email, :password => password }
  save_settings
end


a = Mechanize.new { |agent|
  agent.read_timeout = 40
  agent.user_agent = 'GlideCreate/1 CFNetwork/711.1.16 Darwin/14.0.0'
 
  begin 
    agent.cookie_jar.load(COOKIES)
  rescue Errno::ENOENT
    puts "No cookies to load"
  end
  
  agent.log = Logger.new(STDOUT)
}


def write_data(page)
  puts "JSON: #{page.body}"
  data = JSON.parse(page.body)
  timestamp = Time.now.to_i
  data[:timestamp] = timestamp
  data[:created_on] = Time.at(timestamp).utc.iso8601

  hash_dir_name = (timestamp / 3600).to_i.to_s
  unless Dir.exists?(hash_dir_name)
    Dir.mkdir(hash_dir_name)
  end

  filename = "#{hash_dir_name}/profile_#{timestamp}.json.gz"
  puts "Writing json to #{filename}..."
  Zlib::GzipWriter.open(filename) do |gz|
    gz.write JSON.generate(data)
  end

  puts "...done!"
end

page = a.get(PROFILE_URL)
puts "Initial profile body"
puts page.body

if page.class != Mechanize::File
  page.forms[0].email = $settings[:email]
  page.forms[0].password = $settings[:password]
  confirm_page = page.forms[0].submit

  puts "Post login page"
  puts confirm_page.body

  if confirm_page.body.length == 0
    puts "Trying profile again...."
    page = a.get(PROFILE_URL)
    puts "Page request came back with #{page.class}"
    if page.class == Mechanize::File
      write_data(page)
    else
      puts "Failed to get data after login?"
    end
  else
    puts "Need confirmation?"
    puts "Please type in your ED confirmation code: "
    begin
      code = Timeout.timeout(600) { $stdin.gets }
      code.chomp!
      #code = STDIN.noecho(&:gets).chomp
      confirm_page.forms[0].code = code
      post_confirm_page = confirm_page.forms[0].submit
    rescue
      puts "Timeout for code"
    end
  end
else
  write_data(page)
end

a.cookie_jar.save_as COOKIES, :session => true, :format => :yaml