1_download_film_data.rb

require "open-uri"
require "csv"
require "nokogiri"
require "pry"

csv_string = CSV.generate do |csv|
  # this is the max number of pages with data on the-numbers.com
  (1..5401).step(100).each do |page|
    puts page
    doc = Nokogiri::HTML(open("http://www.the-numbers.com/movie/budgets/all/#{page}").read)
    rows = doc.css("tr:not(:first-child)")
    rows.each do |row|
      csv << row.children.map(&:text).reject { |e| e == "\n" }
    end
  end
end

File.write("films.csv", csv_string)

2_add_director_gender.rb

require "nokogiri"
require "reality"
require "open-uri"
require "pry"

include Reality

def wikipedia_lookup(film, year)
  query = URI.escape("#{film} #{year}")
  url = "https://en.wikipedia.org/w/index.php?search=#{query}&title=Special:Search&profile=default&fulltext=1&searchToken=4eic3ybnnptwcaolvqkzevt5q"
  pages = Nokogiri::HTML(open(url).read).css(".mw-search-result-heading a").map(&:text)

  pages.take(5).each do |page|
    page = Entity(page)
    return page if !page.nil? && !page.directors.nil?
  end

  nil
end

index = -1
CSV.read("films.csv").reject(&:empty?).each do |_, date, film, budget, domestic, world_wide|
  index += 1
  # next if index < 18 # use this to start at a given index

  puts [index, film].join("\t")
  year = date.split("/").last

  if year.to_i > 2017 || [domestic, world_wide].uniq == ["$0"]
    next
  end

  film_page = wikipedia_lookup(film, year) rescue nil

  row = [date, film, budget, domestic, world_wide]

  if film_page.nil?
    puts "ERROR 1"
    puts row = CSV.generate_line(row + [nil, nil])

    open("films_and_directors.csv", "a") { |f| f << row }
  else
    begin
      directors = film_page.directors
      names = directors.map(&:name)
      genders = directors.map(&:sex)

      puts row = CSV.generate_line(row + [names.join(","), genders.join(",")])

      open("films_and_directors.csv", "a") { |f| f << row }
    rescue
      puts "ERROR 2"
      puts row = CSV.generate_line(row + [nil, nil])

      open("films_and_directors.csv", "a") { |f| f << row }
    end
  end
end

3_format_data.rb

require "csv"
require "pry"


def parse_figure(figure)
  Integer(figure.gsub(/[^\d]/, ""))
end

CSV.open("films_and_directors_formatted.csv", "wb") do |csv|
  CSV.read("films_and_directors.csv").each do |date, title, budget, domestic, worldwide, directors, genders|
    next if directors.nil? || genders.nil?
    next if directors.include?(",") || genders.include?(",")

    csv << [
      title,
      date,
      directors,
      parse_figure(budget),
      parse_figure(domestic),
      parse_figure(worldwide),
      genders
    ]
  end
end

4_analysis.rb

require "csv"
require "pry"

class Film
  attr_reader :title, :date, :director, :budget, :domestic, :worldwide, :gender

  def initialize(title, date, director, budget, domestic, worldwide, gender)
    @title, @date, @director, @budget, @domestic, @worldwide, @gender =
      title, date, director, Integer(budget), Integer(domestic), Integer(worldwide), gender
  end

  def domestic_roi
    domestic.to_f / budget
  end

  def worldwide_roi
    worldwide.to_f / budget
  end

  def non_domestic_roi
    (worldwide - domestic).to_f / budget
  end

  def female?
    gender == "female"
  end

  def male?
    gender == "male"
  end
end

data = CSV.read("films_and_directors_formatted.csv")
data.map! { |r| Film.new(*r) }

female = data.select(&:female?)
male = data.select(&:male?)

female_films_used = []
female_roi_means = []
male_films_used = []
male_roi_means = []

metric = ARGV[0].to_sym || :domestic_roi

paired_data = []

female_rois = []
male_rois = []

female.each do |film|
  matched_film = male.select { |f| f.budget == film.budget }.sample
  if matched_film.nil?
    closest_budget = male.min_by { |f| (f.budget - film.budget).abs }.budget
    if closest_budget.between?(film.budget * 0.95, film.budget * 1.05)
      matched_film = male.select { |f| f.budget == closest_budget }.sample
    else
      puts "Failed to match: #{film.title} #{film.budget} (best match: #{closest_budget})"
      next
    end
  end

  paired_data << [film.title, film.domestic_roi, matched_film.title, matched_film.domestic_roi]

  female_rois << film.send(metric)
  male_rois   << matched_film.send(metric)

  male_films_used << matched_film
end

CSV.open("paired_data.csv", "w") { |csv| paired_data.each { |r| csv << r } }

male_roi_mean = male_rois.reduce(&:+) / male_rois.size
female_roi_mean = female_rois.reduce(&:+) / female_rois.size

male_roi_means << male_roi_mean
female_roi_means << female_roi_mean

male_roi_mean = male_roi_means.reduce(&:+) / male_roi_means.size
female_roi_mean = female_roi_means.reduce(&:+) / female_roi_means.size

puts "Metric:               #{metric.to_s.capitalize}"
puts "Female ROI Mean:      #{female_roi_mean}"
puts "Male ROI Mean:        #{male_roi_mean}"
puts "Male Films Covered:   #{male_films_used.uniq.size}"
puts "Female Films Covered: #{female.size}"

films.csv