Pool Predictors

run.sh

#!/usr/bin/env bash

if [ -a /home/ryan/bin/weka/weka-3-7-11/weka.jar ]
then
  export CLASSPATH=/home/ryan/bin/weka/weka-3-7-11/weka.jar:/home/ryan/Source/student-filters/StudentFilters.jar:.
else
  export CLASSPATH=/Applications/weka-3-7-11/weka.jar:.
fi

if [ "$(uname)" == "Darwin" ]
then
  export CPUS=$(sysctl hw.ncpu | awk '{print $2}')
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]
then
  export CPUS=$(grep -c ^processor /proc/cpuinfo)
fi

echo -e "There are $CPUS cpu(s) on this machine."
echo $CLASSPATH

time java -Xmx8192M weka.filters.supervised.attribute.AttributeSelection -S "weka.attributeSelection.Ranker -N -1" \
  -E "weka.attributeSelection.PrincipalComponents -R 0.99 -A 100" \
  -b \
  -i ~/Source/cs7641/project1/arff/nfl-training.csv.arff \
  -o ~/Desktop/nfl-training-week-12-pca-099.csv.arff \
  -r ~/Desktop/nfl-2014-week-12.arff \
  -s ~/Desktop/nfl-2014-week-12-pca-099.arff \
  -c last

time java -Xmx8192M weka.classifiers.functions.MultilayerPerceptron \
  -L 0.3 -M 0.2 -N 500 -V 80 -S 0 -E 20 -H i -output-debug-info \
  -t ~/Desktop/nfl-training-week-12-pca-099.csv.arff \
  -T ~/Desktop/nfl-2014-week-12-pca-099.arff \
  -c last -p 0

time java -Xmx8192M weka.filters.unsupervised.attribute.AddCluster \
  -W "weka.clusterers.SimpleKMeans -init 0 -N 2 -A \"weka.core.EuclideanDistance -R first-last\" -I 1000 -O -num-slots $CPUS" \
  -b \
  -i ~/Source/cs7641/project1/arff/nfl-training.csv.arff \
  -o ~/Desktop/nfl-training-clustered.arff \
  -r ~/Desktop/nfl-2014-week-12.arff \
  -s ~/Desktop/nfl-2014-week-12-clustered.arff

time java -Xmx8192M weka.classifiers.functions.MultilayerPerceptron \
  -L 0.3 -M 0.2 -N 500 -V 80 -S 0 -E 20 -H i -output-debug-info \
  -t ~/Desktop/nfl-training-clustered.arff \
  -T ~/Desktop/nfl-2014-week-12-clustered.arff \
  -c last -p 0

upcoming_nfl_to_csv.rb

require 'sports_data_api'
require 'nokogiri'
require 'geocoder'
require 'csv'
require 'pstore'
require 'time'
require 'mechanize'
require './boxscore'
require './team'

# CSV Column Names
names = %w(
  DayOfWeek
  Month
  VegasLine
  OverUnder
  TeamAIsHome
  TeamAOffensiveScheme
  TeamADefensiveScheme
  TeamAOffensivePtsRank
  TeamAOffensiveYdsRank
  TeamAOffensivePly
  TeamAOffensiveYdsPerPlay
  TeamAOffensiveTORank
  TeamAOffensiveFLRank
  TeamAOffensive1stPy
  TeamAOffensive1stDRank
  TeamAOffensivePassingCmp
  TeamAOffensivePassingAttRank
  TeamAOffensivePassingYdsRank
  TeamAOffensivePassingTDRank
  TeamAOffensivePassingIntRank
  TeamAOffensivePassingNYPerAttRank
  TeamAOffensivePassing1stD
  TeamAOffensiveRushingAttRank
  TeamAOffensiveRushingYdsRank
  TeamAOffensiveRushingTDRank
  TeamAOffensiveRushingYPerARank
  TeamAOffensiveRushing1stD
  TeamADefensivePtsRank
  TeamADefensiveYdsRank
  TeamADefensivePly
  TeamADefensiveYdsPerPlay
  TeamADefensiveTORank
  TeamADefensiveFLRank
  TeamADefensive1stPy
  TeamADefensive1stDRank
  TeamADefensivePassingCmp
  TeamADefensivePassingAttRank
  TeamADefensivePassingYdsRank
  TeamADefensivePassingTDRank
  TeamADefensivePassingIntRank
  TeamADefensivePassingNYPerAttRank
  TeamADefensivePassing1stD
  TeamADefensiveRushingAttRank
  TeamADefensiveRushingYdsRank
  TeamADefensiveRushingTDRank
  TeamADefensiveRushingYPerARank
  TeamADefensiveRushing1stD
  TeamBIsHome
  TeamBOffensiveScheme
  TeamBDefensiveScheme
  TeamBOffensivePtsRank
  TeamBOffensiveYdsRank
  TeamBOffensivePly
  TeamBOffensiveYdsPerPlay
  TeamBOffensiveTORank
  TeamBOffensiveFLRank
  TeamBOffensive1stPy
  TeamBOffensive1stDRank
  TeamBOffensivePassingCmp
  TeamBOffensivePassingAttRank
  TeamBOffensivePassingYdsRank
  TeamBOffensivePassingTDRank
  TeamBOffensivePassingIntRank
  TeamBOffensivePassingNYPerAttRank
  TeamBOffensivePassing1stD
  TeamBOffensiveRushingAttRank
  TeamBOffensiveRushingYdsRank
  TeamBOffensiveRushingTDRank
  TeamBOffensiveRushingYPerARank
  TeamBOffensiveRushing1stD
  TeamBDefensivePtsRank
  TeamBDefensiveYdsRank
  TeamBDefensivePly
  TeamBDefensiveYdsPerPlay
  TeamBDefensiveTORank
  TeamBDefensiveFLRank
  TeamBDefensive1stPy
  TeamBDefensive1stDRank
  TeamBDefensivePassingCmp
  TeamBDefensivePassingAttRank
  TeamBDefensivePassingYdsRank
  TeamBDefensivePassingTDRank
  TeamBDefensivePassingIntRank
  TeamBDefensivePassingNYPerAttRank
  TeamBDefensivePassing1stD
  TeamBDefensiveRushingAttRank
  TeamBDefensiveRushingYdsRank
  TeamBDefensiveRushingTDRank
  TeamBDefensiveRushingYPerARank
  TeamBDefensiveRushing1stD
  Outcome
)

CSV.open(File.join('.', 'data', 'predictions.csv'), 'w') do |csv|
  csv << names
  agent = Mechanize.new
  year = ARGV[0].to_i
  week = ARGV[1].to_i
  page = agent.get("http://www.pro-football-reference.com/years/#{year}/games.htm")

  game_tds = page.parser.xpath("//*[@id=\"games_left\"]/tbody/tr/td[1][text()=\"#{week}\"]/.. | //*[@id=\"games\"]//tr[td[1][string(number(text()))=\"#{week}\"] and td[4]/node()]")
  puts "#{year}:#{week} has #{game_tds.length} games."
  game_tds.each do |game_td|
    children_count = game_td.xpath('td').count
    if children_count === 8
      # Has yet to be played
      date = Time.parse(game_td.xpath('./td[3]/a/text()').first.to_s)
      away_uri = page.uri.merge(game_td.xpath('./td[4]/a/@href').first.value)
      home_uri = page.uri.merge(game_td.xpath('./td[6]/a/@href').first.value)
      boxscore_uri = page.uri.merge(game_td.xpath('./td[3]/a/@href').first.value)
      away_page = agent.get(away_uri)
      home_page = agent.get(home_uri)
      boxscore_page = agent.get(boxscore_uri)
    else
      # Has been played
      date = Time.parse(game_td.xpath('./td[3]/@csk').first.value)
      boxscore_uri = page.uri.merge(game_td.xpath('./td[4]/a/@href').first.value)
      boxscore_page = agent.get(boxscore_uri)
      winner_uri = page.uri.merge(game_td.xpath('./td[5]//a/@href').first.value)
      looser_uri = page.uri.merge(game_td.xpath('./td[7]//a/@href').first.value)
      winner_page = agent.get(winner_uri)
      looser_page = agent.get(looser_uri)
      if game_td.xpath('./td[6]').first.content === '@'
        away_page = winner_page
        home_page = looser_page
      else
        home_page = winner_page
        away_page = looser_page
      end
    end

    away_team = Team.new(away_page)
    home_team = Team.new(home_page)
    boxscore = Boxscore.new(boxscore_page)
    puts "[#{year}:#{week}:#{date}] #{boxscore.vegas_line} #{nil}; Home: #{home_team.team_name} vs Away: #{away_team.team_name}"
    csv << [
      date.strftime('%^a'),
      date.strftime('%^b'),
      boxscore.vegas_line,
      nil,
      true,
      home_team.offensive_scheme,
      home_team.defensive_scheme,
      home_team.points_scored_rank(:offense),
      home_team.total_yards_rank(:offense),
      home_team.play_count(:offense),
      home_team.yards_per_play(:offense),
      home_team.turnovers_lost_rank(:offense),
      home_team.fumbles_lost_rank(:offense),
      home_team.first_down_from_penalty(:offense),
      home_team.first_down_rank(:offense),
      home_team.passing_completions(:offense),
      home_team.passing_attempts_rank(:offense),
      home_team.passing_yards_rank(:offense),
      home_team.passing_td_rank(:offense),
      home_team.passing_int_rank(:offense),
      home_team.net_yards_per_attempt_rank(:offense),
      home_team.first_down_from_passing(:offense),
      home_team.rushing_attempts_rank(:offense),
      home_team.rushing_yards_rank(:offense),
      home_team.rushing_tds_rank(:offense),
      home_team.rushing_yards_per_attempt_rank(:offense),
      home_team.rushing_first_downs(:offense),
      home_team.points_scored_rank(:defense),
      home_team.total_yards_rank(:defense),
      home_team.play_count(:defense),
      home_team.yards_per_play(:defense),
      home_team.turnovers_lost_rank(:defense),
      home_team.fumbles_lost_rank(:defense),
      home_team.first_down_from_penalty(:defense),
      home_team.first_down_rank(:defense),
      home_team.passing_completions(:defense),
      home_team.passing_attempts_rank(:defense),
      home_team.passing_yards_rank(:defense),
      home_team.passing_td_rank(:defense),
      home_team.passing_int_rank(:defense),
      home_team.net_yards_per_attempt_rank(:defense),
      home_team.first_down_from_passing(:defense),
      home_team.rushing_attempts_rank(:defense),
      home_team.rushing_yards_rank(:defense),
      home_team.rushing_tds_rank(:defense),
      home_team.rushing_yards_per_attempt_rank(:defense),
      home_team.rushing_first_downs(:defense),
      false,
      away_team.offensive_scheme,
      away_team.defensive_scheme,
      away_team.points_scored_rank(:offense),
      away_team.total_yards_rank(:offense),
      away_team.play_count(:offense),
      away_team.yards_per_play(:offense),
      away_team.turnovers_lost_rank(:offense),
      away_team.fumbles_lost_rank(:offense),
      away_team.first_down_from_penalty(:offense),
      away_team.first_down_rank(:offense),
      away_team.passing_completions(:offense),
      away_team.passing_attempts_rank(:offense),
      away_team.passing_yards_rank(:offense),
      away_team.passing_td_rank(:offense),
      away_team.passing_int_rank(:offense),
      away_team.net_yards_per_attempt_rank(:offense),
      away_team.first_down_from_passing(:offense),
      away_team.rushing_attempts_rank(:offense),
      away_team.rushing_yards_rank(:offense),
      away_team.rushing_tds_rank(:offense),
      away_team.rushing_yards_per_attempt_rank(:offense),
      away_team.rushing_first_downs(:offense),
      away_team.points_scored_rank(:defense),
      away_team.total_yards_rank(:defense),
      away_team.play_count(:defense),
      away_team.yards_per_play(:defense),
      away_team.turnovers_lost_rank(:defense),
      away_team.fumbles_lost_rank(:defense),
      away_team.first_down_from_penalty(:defense),
      away_team.first_down_rank(:defense),
      away_team.passing_completions(:defense),
      away_team.passing_attempts_rank(:defense),
      away_team.passing_yards_rank(:defense),
      away_team.passing_td_rank(:defense),
      away_team.passing_int_rank(:defense),
      away_team.net_yards_per_attempt_rank(:defense),
      away_team.first_down_from_passing(:defense),
      away_team.rushing_attempts_rank(:defense),
      away_team.rushing_yards_rank(:defense),
      away_team.rushing_tds_rank(:defense),
      away_team.rushing_yards_per_attempt_rank(:defense),
      away_team.rushing_first_downs(:defense),
      nil
    ]
  end
end