Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
ED Scraper (ITS RUBY!)
require 'mechanize'
require 'logger'
require 'io/console'
require 'json'
SETTINGS = ".ed_profile_scraper.yml"
COOKIES = ".ed_profile_scraper.cookies"
BASE_URL = "https://companion.orerve.net/"
LOGIN_URL = BASE_URL + "user/login"
CONFIRM_URL = BASE_URL + "user/confirm"
PROFILE_URL = BASE_URL + "profile"
def save_settings
puts "Saving settings..."
File.open(SETTINGS, "w") do |file|
file.write $settings.to_yaml
end
puts "...done!"
end
begin
$settings = YAML::load_file SETTINGS
rescue Errno::ENOENT
puts "No settings found!"
puts "Please type in your ED username (email): "
email = gets.chomp
puts "Please type in your ED password: "
password = STDIN.noecho(&:gets).chomp
$settings = { :email => email, :password => password }
save_settings
end
a = Mechanize.new { |agent|
agent.read_timeout = 40
agent.user_agent = 'GlideCreate/1 CFNetwork/711.1.16 Darwin/14.0.0'
begin
agent.cookie_jar.load(COOKIES)
rescue Errno::ENOENT
puts "No cookies to load"
end
agent.log = Logger.new(STDOUT)
}
def write_data(page)
puts "JSON: #{page.body}"
data = JSON.parse(page.body)
timestamp = Time.now.to_i
data[:timestamp] = timestamp
data[:created_on] = Time.at(timestamp).utc.iso8601
hash_dir_name = (timestamp / 3600).to_i.to_s
unless Dir.exists?(hash_dir_name)
Dir.mkdir(hash_dir_name)
end
filename = "#{hash_dir_name}/profile_#{timestamp}.json.gz"
puts "Writing json to #{filename}..."
Zlib::GzipWriter.open(filename) do |gz|
gz.write JSON.generate(data)
end
puts "...done!"
end
page = a.get(PROFILE_URL)
puts "Initial profile body"
puts page.body
if page.class != Mechanize::File
page.forms[0].email = $settings[:email]
page.forms[0].password = $settings[:password]
confirm_page = page.forms[0].submit
puts "Post login page"
puts confirm_page.body
if confirm_page.body.length == 0
puts "Trying profile again...."
page = a.get(PROFILE_URL)
puts "Page request came back with #{page.class}"
if page.class == Mechanize::File
write_data(page)
else
puts "Failed to get data after login?"
end
else
puts "Need confirmation?"
puts "Please type in your ED confirmation code: "
begin
code = Timeout.timeout(600) { $stdin.gets }
code.chomp!
#code = STDIN.noecho(&:gets).chomp
confirm_page.forms[0].code = code
post_confirm_page = confirm_page.forms[0].submit
rescue
puts "Timeout for code"
end
end
else
write_data(page)
end
a.cookie_jar.save_as COOKIES, :session => true, :format => :yaml
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment