amtrak

amtrak.rb

#!/usr/bin/env ruby

require 'rubygems'
require 'cgi'
require 'httparty'
require 'nokogiri'

@url = "https://tickets.amtrak.com/itd/amtrak"

@train = '523'
@station = 'OAC'
@date = DateTime.now.strftime('%m/%d/%Y')
@selection = 'arrivalTime' # or departTime

class Amtrak
  include HTTParty
  base_uri 'https://tickets.amtrak.com'
	@log_cache = true
	class << self
		attr_accessor :log_cache
	end
  # debug_output
  # headers({
  #   "DNT" => "1",
  #   "Content-Type" => "application/x-www-form-urlencoded",
  #   "Referer" => "https://tickets.amtrak.com/itd/amtrak",
  #   "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
  #   "User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.3.12 (KHTML, like Gecko) Version/10.0.2 Safari/602.3.12",
  #   "Origin" => "https://tickets.amtrak.com"
  # })
  
	Stations = %w{ COX ARN RLN RSV SAC DAV SUI MTZ RIC BKY EMY OKJ OAC HAY FMT GAC SCC SJC }

	SAC_OKJ = Stations[4..11]
	SAC_OAC = Stations[4..12]
	SAC_ALL = Stations[4..-1]
	ARN_OKJ = Stations[1..11]

	WeekdayTrains = {
		521 => SAC_ALL,
		523 => SAC_ALL,
		525 => SAC_ALL,
		527 => SAC_ALL,
		529 => ARN_OKJ,
		531 => SAC_OAC,
		535 => SAC_OAC,
		537 => SAC_ALL,
		541 => SAC_OAC,
		543 => SAC_ALL,
		545 => SAC_OAC,
		547 => SAC_ALL,
		549 => SAC_OAC,
		551 => SAC_OKJ,
		553 => SAC_OKJ,

		520 => SAC_OKJ,
		522 => SAC_OKJ,
		524 => SAC_ALL,
		528 => SAC_ALL,
		530 => SAC_OKJ,
		532 => SAC_ALL,
		534 => SAC_OKJ,
		536 => ARN_OKJ,
		538 => SAC_ALL,
		540 => SAC_OKJ,
		542 => SAC_ALL,
		544 => SAC_OKJ,
		546 => SAC_ALL,
		548 => SAC_ALL,
		550 => SAC_OAC,
	}
	
  def self.arrival train, station, date, opts = { use_cache: true }
    # selection = 'arrivalTime' # or departTime
		
		return Arrival.new(nil, nil) unless WeekdayTrains[train].include?(station)
    
		date_str = date.strftime('%m/%d/%Y')
		
		cache_key = "#{date.strftime('%Y-%m-%d')}.#{train}.#{station}"
		cache_file = "#{File.dirname(__FILE__)}/cache/#{cache_key}.html"
		if opts[:use_cache] && File.exists?(cache_file)
			puts "Using cached file #{cache_file}..." if @log_cache
			return Arrival.parse(File.open(cache_file).read)
		else
			sleep 1 # avoid rate-limiting errors
		end
		
    fields = {
      'requestor' => 'amtrak.presentation.handler.page.rail.AmtrakRailGetTrainStatusPageHandler',
      CGI.escape("/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/@trainStatusType") => 'statusByTrainNumber',
      # 'xwdf_SortBy' => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate/@radioSelect",
      # 'wdf_SortBy' => selection,
      # 'xwdf_origin' => "/sessionWorkflow/productWorkflow[@product='Rail']/travelSelection/journeySelection[1]/departLocation/search",
      # 'wdf_origin' => '',
      'xwdf_destination' => "/sessionWorkflow/productWorkflow[@product='Rail']/travelSelection/journeySelection[1]/arriveLocation/search",
      'wdf_destination' => station,
      'xwdf_trainNumber' => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/segmentRequirements[1]/serviceCode",
      'wdf_trainNumber' => train,
      CGI.escape("/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate.usdate") => date_str,
      CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail']") => '',
      # CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail'].x") => 79,
      # CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail'].y") => 16
    }
    
    result = post("/itd/amtrak", { body: fields, headers: nil })
		
		if opts[:use_cache]
			puts "Caching #{cache_file}..." if @log_cache
			`mkdir -p cache`
			File.open(cache_file, 'w') { |f| f << result.body }
		end
		
		return Arrival.parse(result)
  end
end



# @fields = {
#   'requestor' => 'amtrak.presentation.handler.page.rail.AmtrakRailGetTrainStatusPageHandler',
#   CGI.escape("/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/@trainStatusType") => 'statusByTrainNumber',
#   'xwdf_SortBy' => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate/@radioSelect",
#   'wdf_SortBy' => @selection,
#   'xwdf_origin' => "/sessionWorkflow/productWorkflow[@product='Rail']/travelSelection/journeySelection[1]/departLocation/search",
#   'wdf_origin' => '',
#   'xwdf_destination' => "/sessionWorkflow/productWorkflow[@product='Rail']/travelSelection/journeySelection[1]/arriveLocation/search",
#   'wdf_destination' => @station,
#   'xwdf_trainNumber' => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/segmentRequirements[1]/serviceCode",
#   'wdf_trainNumber' => @train,
#   CGI.escape("/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate.usdate") => @date,
#   CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail']") => '',
#   CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail'].x") => 79,
#   CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail'].y") => 16
# }

# @fields = {
#   "requestor" => "amtrak.presentation.handler.page.rail.AmtrakRailGetTrainStatusPageHandler",
#   CGI.escape("/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/@trainStatusType") => "statusByTrainNumber",
#   "xwdf_SortBy" => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate/@radioSelect",
#   "wdf_SortBy" => "arrivalTime",
#   "xwdf_SortBy" => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate/@radioSelect",
#   "xwdf_origin" => "/sessionWorkflow/productWorkflow[@product='Rail']/travelSelection/journeySelection[1]/departLocation/search",
#   "wdf_origin" => "",
#   "xwdf_destination" => "/sessionWorkflow/productWorkflow[@product='Rail']/travelSelection/journeySelection[1]/arriveLocation/search",
#   "wdf_destination" => "Oakland - Coliseum Airport, CA (OAC)",
#   "xwdf_trainNumber" => "/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/segmentRequirements[1]/serviceCode",
#   "wdf_trainNumber" => "525",
#   CGI.escape("/sessionWorkflow/productWorkflow[@product='Rail']/tripRequirements/journeyRequirements[1]/departDate.usdate") => "02/14/2017",
#   CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail']") => "",
#   CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail'].x") => "79",
#   CGI.escape("_handler=amtrak.presentation.handler.request.rail.AmtrakRailTrainStatusSearchRequestHandler/_xpath=/sessionWorkflow/productWorkflow[@product='Rail'].y") => "16",
# }



# @result = HTTParty.get(@url)
# @result = Amtrak.post("/itd/amtrak", { body: @fields, headers: @headers })

# @result = Amtrak.arrival(@train, @station, @date)
# @document = Nokogiri::HTML(@result)

class Arrival
	attr_accessor :scheduled, :actual
	
	def initialize scheduled, actual
		self.scheduled = scheduled
		self.actual = actual
	end
	
	def self.parse html
		doc = Nokogiri::HTML(html)
		
		scheduled_elems = doc.css('.train-status-schedule-block_details_info_secondary')
		actual_elems = doc.css('.train-status-schedule-block_details_time')
    scheduled_regex = /Scheduled[^\w](.+)/
    
    # try legacy parsing (pre 2017-10-09)
		if scheduled_elems.length < 1
  		scheduled_elems = doc.css('.scheduledArriveDepartMsg')
  		actual_elems = doc.css('.arriveDepartTime')
      scheduled_regex = /Scheduled Arrival[^\w]+(.+)/
    end

		if scheduled_elems.length < 1
			return Arrival.new nil, nil
		end

		scheduled_txt = scheduled_elems[0].text.strip
		scheduled = DateTime.parse(/Scheduled[^\w+](.+)/.match(scheduled_txt)[1])

		result = Arrival.new(scheduled, nil)

		if actual_elems.length > 0
			actual_txt = actual_elems[0].text
			result.actual = DateTime.parse(actual_txt)
		end

		return result
	end
	
	def delay
		if actual
			return ((actual - scheduled) * 24 * 60).to_i
		else
			return nil
		end
	end

	def to_s
		time_str = scheduled.strftime('%H:%M')
		delay_str = ""
		if actual
			time_str = actual.strftime('%H:%M')
			delay_str = "\t(#{format('%+d', delay)})"
		end
		return "#{time_str}#{delay_str}"
	end
end

def print_train station, train, date, use_cache = true
  arrival = Amtrak.arrival(train, station, date, use_cache: use_cache)
	if arrival.scheduled != nil
  	puts "#{train}\t#{station}\t#{arrival}"
	else
		puts "(NA)"
	end
end



# %w{EMY OKJ OAC}.each do |station|
# %w{OAC}.each do |station|
#   print_morning_trains station
# end



#date = DateTime.now
#print_train 'OAC', 523, date
#puts weekday_trains.inspect
# puts weekday_trains.map{|k,v| v.length}.inject(0){|m,v| m + v}
#puts weekday_trains[523]

# (19..21).each do |day|
#	date = DateTime.parse("2017-04-#{day}")

# end

cache_yesterday.rb

#!/usr/bin/env ruby

require_relative 'amtrak'

yesterday = DateTime.now - 1

Amtrak::WeekdayTrains.each do |train, stops|
	puts ""
	stops.each do |stop|
		print_train stop, train, yesterday
	end
end

gen_datafiles.rb

#!/usr/bin/env ruby

require_relative 'amtrak'

Amtrak.log_cache = false

@start_date = DateTime.new(2017, 04, 19)
@end_date = DateTime.new(2017, 05, 30)

@dates = (@start_date..@end_date).reject{|d| d.sunday? || d.saturday? || d.strftime('%Y-%m-%d') == '2017-05-29'}

ColiseumPriors = {
  521 => [
    # [521, 'SAC'],
    [521, 'DAV'],
    [521, 'SUI'],
    [521, 'MTZ'],
    [521, 'RIC'],
    [521, 'BKY'],
    [521, 'EMY'],
    [521, 'OKJ'],
  
    # [520, 'OKJ'],
    [520, 'EMY'],
    [520, 'BKY'],
    [520, 'RIC']
  ],
  523 => [
    [521, 'OAC'],
    
    [523, 'DAV'],
    [523, 'SUI'],
    [523, 'MTZ'],
    [523, 'RIC'],
    [523, 'BKY'],
    [523, 'EMY'],
    [523, 'OKJ'],
  
    [522, 'EMY'],
    [522, 'BKY'],
    [522, 'RIC']
  ],
  525 => [
    [521, 'OAC'],
    [523, 'OAC'],
    
    [525, 'DAV'],
    [525, 'SUI'],
    [525, 'MTZ'],
    [525, 'RIC'],
    [525, 'BKY'],
    [525, 'EMY'],
    [525, 'OKJ'],
  
    [524, 'SJC'],
    [524, 'SCC'],
    [524, 'GAC'],
    [524, 'FMT'],
    [524, 'HAY'],
    [524, 'OAC'],
    [524, 'OKJ'],
    [524, 'EMY'],
    [524, 'BKY']
  ],
  527 => [
    [521, 'OAC'],
    [523, 'OAC'],
    [525, 'OAC'],
    
    [527, 'DAV'],
    [527, 'SUI'],
    [527, 'MTZ'],
    [527, 'RIC'],
    [527, 'BKY'],
    [527, 'EMY'],
    [527, 'OKJ'],
  ]
}

def write_matrix(mat, fname)
  open(fname, 'w') do |f|
    f << mat.map{|p| p.map(&:to_s).join(",")}.join("\n")
  end
end

ColiseumPriors.each do |train, pairs|
  puts train
  
  priors = @dates.map do |date|
    pairs.map do |pair|
      Amtrak.arrival(pair.first, pair.last, date).delay
    end
  end
  arrivals = @dates.map do |date|
    [Amtrak.arrival(train, 'OAC', date).delay]
  end
  
  write_matrix(priors, "data/#{train}-priors.csv")
  write_matrix(arrivals, "data/#{train}-output.csv")
end

morning_trains.rb

#!/usr/bin/env ruby

require './amtrak.rb'

def print_morning_trains station, date = DateTime.now
  puts station
  trains = [521, 523, 525, 527]
  trains.each do |train|
    print_train station, train, date, false
  end
  puts
end

print_morning_trains 'OAC'

train_matrix.rb

#!/usr/bin/env ruby

require 'date'

require_relative 'amtrak'

date = DateTime.parse ARGV[0]

Amtrak.log_cache = false

puts "\t" + Amtrak::Stations.join("\t")
Amtrak::WeekdayTrains.sort.each do |train, stops|
	arrivals = Amtrak::Stations.map{|stop| Amtrak.arrival(train, stop, date)}
	delays = arrivals.map(&:delay)
	puts ([train] + delays).join("\t")
end