Create a gist now

Instantly share code, notes, and snippets.

@dschep /README.md
Last active Dec 14, 2015

What would you like to do?
Bechdel Test Passing Rates

This is computed from the results on http://bechdeltest.com/

The data is very sparse before 1930ish, leading to some 100% pass rates.

[{"value": 0.0, "year": 1900}, {"value": 0.0, "year": 1902}, {"value": 0.0, "year": 1903}, {"value": 0.0, "year": 1906}, {"value": 1.0, "year": 1910}, {"value": 0.0, "year": 1912}, {"value": 0.0, "year": 1915}, {"value": 1.0, "year": 1916}, {"value": 0.6, "year": 1917}, {"value": 1.0, "year": 1918}, {"value": 0.0, "year": 1919}, {"value": 0.5, "year": 1920}, {"value": 0.5, "year": 1921}, {"value": 0.2, "year": 1922}, {"value": 0.0, "year": 1923}, {"value": 0.1111111111111111, "year": 1924}, {"value": 0.0, "year": 1925}, {"value": 0.2, "year": 1926}, {"value": 0.2222222222222222, "year": 1927}, {"value": 0.08333333333333333, "year": 1928}, {"value": 0.0, "year": 1929}, {"value": 0.42857142857142855, "year": 1930}, {"value": 0.46153846153846156, "year": 1931}, {"value": 0.1, "year": 1932}, {"value": 0.42857142857142855, "year": 1933}, {"value": 0.1111111111111111, "year": 1934}, {"value": 0.4444444444444444, "year": 1935}, {"value": 0.4375, "year": 1936}, {"value": 0.09090909090909091, "year": 1937}, {"value": 0.3333333333333333, "year": 1938}, {"value": 0.2608695652173913, "year": 1939}, {"value": 0.35294117647058826, "year": 1940}, {"value": 0.35294117647058826, "year": 1941}, {"value": 0.3333333333333333, "year": 1942}, {"value": 0.45454545454545453, "year": 1943}, {"value": 0.5, "year": 1944}, {"value": 0.5555555555555556, "year": 1945}, {"value": 0.35714285714285715, "year": 1946}, {"value": 0.4444444444444444, "year": 1947}, {"value": 0.26666666666666666, "year": 1948}, {"value": 0.6363636363636364, "year": 1949}, {"value": 0.375, "year": 1950}, {"value": 0.5, "year": 1951}, {"value": 0.2857142857142857, "year": 1952}, {"value": 0.4666666666666667, "year": 1953}, {"value": 0.1875, "year": 1954}, {"value": 0.35714285714285715, "year": 1955}, {"value": 0.35714285714285715, "year": 1956}, {"value": 0.2608695652173913, "year": 1957}, {"value": 0.35294117647058826, "year": 1958}, {"value": 0.6875, "year": 1959}, {"value": 0.4444444444444444, "year": 1960}, {"value": 0.26666666666666666, "year": 1961}, {"value": 0.5263157894736842, "year": 1962}, {"value": 0.2, "year": 1963}, {"value": 0.16666666666666666, "year": 1964}, {"value": 0.5, "year": 1965}, {"value": 0.3181818181818182, "year": 1966}, {"value": 0.5, "year": 1967}, {"value": 0.34782608695652173, "year": 1968}, {"value": 0.29411764705882354, "year": 1969}, {"value": 0.5, "year": 1970}, {"value": 0.38095238095238093, "year": 1971}, {"value": 0.47058823529411764, "year": 1972}, {"value": 0.3888888888888889, "year": 1973}, {"value": 0.38461538461538464, "year": 1974}, {"value": 0.3888888888888889, "year": 1975}, {"value": 0.4, "year": 1976}, {"value": 0.5, "year": 1977}, {"value": 0.3684210526315789, "year": 1978}, {"value": 0.16666666666666666, "year": 1979}, {"value": 0.4166666666666667, "year": 1980}, {"value": 0.3333333333333333, "year": 1981}, {"value": 0.39285714285714285, "year": 1982}, {"value": 0.47058823529411764, "year": 1983}, {"value": 0.4230769230769231, "year": 1984}, {"value": 0.5121951219512195, "year": 1985}, {"value": 0.375, "year": 1986}, {"value": 0.5116279069767442, "year": 1987}, {"value": 0.5641025641025641, "year": 1988}, {"value": 0.43243243243243246, "year": 1989}, {"value": 0.5142857142857142, "year": 1990}, {"value": 0.46511627906976744, "year": 1991}, {"value": 0.4473684210526316, "year": 1992}, {"value": 0.5, "year": 1993}, {"value": 0.4745762711864407, "year": 1994}, {"value": 0.47761194029850745, "year": 1995}, {"value": 0.5373134328358209, "year": 1996}, {"value": 0.5303030303030303, "year": 1997}, {"value": 0.42857142857142855, "year": 1998}, {"value": 0.42857142857142855, "year": 1999}, {"value": 0.4875, "year": 2000}, {"value": 0.5595238095238095, "year": 2001}, {"value": 0.5, "year": 2002}, {"value": 0.5444444444444444, "year": 2003}, {"value": 0.5446428571428571, "year": 2004}, {"value": 0.5350877192982456, "year": 2005}, {"value": 0.5352112676056338, "year": 2006}, {"value": 0.546583850931677, "year": 2007}, {"value": 0.5412371134020618, "year": 2008}, {"value": 0.4574898785425101, "year": 2009}, {"value": 0.458955223880597, "year": 2010}, {"value": 0.47104247104247104, "year": 2011}, {"value": 0.46551724137931033, "year": 2012}, {"value": 0.5, "year": 2013}]
#!/usr/bin/env python
"""
Really nasty little script to scrape the data off of bechdeltest.com
"""
import json
import requests
from bs4 import BeautifulSoup, Tag
resp = requests.get('http://bechdeltest.com/', params={'list': 'all'})
soup = BeautifulSoup(resp.content)
movie_list = soup.find(class_='list')
years = {}
year = None
for child in movie_list:
if not isinstance(child, Tag):
continue # ignore text space
try:
assert child.a['id'].startswith('year-')
except (AssertionError, TypeError, KeyError):
pass
else:
year = int(child.a['id'][len('year-'):])
years[year] = [0, 0] #pass/fail
try:
assert 'movie' in child['class']
except (AssertionError, KeyError):
pass
else:
years[year][int(child.img['src'] != '/static/ok.png')] += 1
for year in years:
y, n = years[year]
years[year] = y/float(y+n)
with open('data.json', 'w') as out:
json.dump([{'year': y, 'value': v} for y,v in years.items()], out)
<!DOCTYPE html>
<html>
<meta charset="utf-8">
<head>
<title></title>
<style>
.chart rect {
stroke: white;
fill: steelblue;
}
.chart text {
font: 10px sans-serif;
}
</style>
<script src="http://d3js.org/d3.v3.min.js"></script>
</head>
<body>
<script>
var h = 500 - 40,
w = 900 - 40;
var y = d3.scale.linear()
.domain([0, 1])
.range([0, h-20]);
var x = d3.scale.linear()
.domain([1900, 2013])
.range([0, w]);
var chart = d3.select("body").append("svg")
.attr("class", "chart")
.attr("height", h + 40)
.attr("width", w + 40)
.append('g')
.attr('transform', 'translate(20, 20)');
chart.selectAll('.tickline')
.data(y.ticks(10))
.enter().append('line')
.attr('class', 'tickline')
.attr('x1', 0)
.attr('x2', w)
.attr('y1', function(d) { return h - y(d); })
.attr('y2', function(d) { return h - y(d); })
.style("stroke-width", 1)
.style("stroke", "#ccc");
chart.selectAll('.percent')
.data(y.ticks(10))
.enter().append('text')
.attr('class', 'percent')
.attr('x', 20)
.attr('y', function(d) { return h - y(d); })
.text(String);
d3.json('data.json', function(error, data){
chart.selectAll("rect")
.data(data)
.enter().append("rect")
.attr("x", function(d) { return x(d.year); })
.attr('y', function(d) { return h - y(d.value); })
.attr('height', function(d) { return y(d.value); })
.attr("width", '4');
chart.append('line')
.attr('x1', 0)
.attr('x2', w)
.attr('y1', h)
.attr('y2', h)
.style("stroke-width", 2)
.style("stroke", "#000");
chart.selectAll('.rule')
.data(x.ticks(10))
.enter().append('text')
.attr('class', 'rule')
.attr('x', x)
.attr('y', h)
.attr('dy', 20)
.attr('text-anchor', 'middle')
.text(String)
})
</script>
</body>
</html>
beautifulsoup4
lxml
requests
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment