Skip to content

Instantly share code, notes, and snippets.

@dschep dschep/README.md
Last active Dec 14, 2015

Embed
What would you like to do?
Bechdel Test Passing Rates

This is computed from the results on http://bechdeltest.com/

The data is very sparse before 1930ish, leading to some 100% pass rates.

[{"value": 0.0, "year": 1900}, {"value": 0.0, "year": 1902}, {"value": 0.0, "year": 1903}, {"value": 0.0, "year": 1906}, {"value": 1.0, "year": 1910}, {"value": 0.0, "year": 1912}, {"value": 0.0, "year": 1915}, {"value": 1.0, "year": 1916}, {"value": 0.6, "year": 1917}, {"value": 1.0, "year": 1918}, {"value": 0.0, "year": 1919}, {"value": 0.5, "year": 1920}, {"value": 0.5, "year": 1921}, {"value": 0.2, "year": 1922}, {"value": 0.0, "year": 1923}, {"value": 0.1111111111111111, "year": 1924}, {"value": 0.0, "year": 1925}, {"value": 0.2, "year": 1926}, {"value": 0.2222222222222222, "year": 1927}, {"value": 0.08333333333333333, "year": 1928}, {"value": 0.0, "year": 1929}, {"value": 0.42857142857142855, "year": 1930}, {"value": 0.46153846153846156, "year": 1931}, {"value": 0.1, "year": 1932}, {"value": 0.42857142857142855, "year": 1933}, {"value": 0.1111111111111111, "year": 1934}, {"value": 0.4444444444444444, "year": 1935}, {"value": 0.4375, "year": 1936}, {"value": 0.09090909090909091, "year": 1937}, {"value": 0.3333333333333333, "year": 1938}, {"value": 0.2608695652173913, "year": 1939}, {"value": 0.35294117647058826, "year": 1940}, {"value": 0.35294117647058826, "year": 1941}, {"value": 0.3333333333333333, "year": 1942}, {"value": 0.45454545454545453, "year": 1943}, {"value": 0.5, "year": 1944}, {"value": 0.5555555555555556, "year": 1945}, {"value": 0.35714285714285715, "year": 1946}, {"value": 0.4444444444444444, "year": 1947}, {"value": 0.26666666666666666, "year": 1948}, {"value": 0.6363636363636364, "year": 1949}, {"value": 0.375, "year": 1950}, {"value": 0.5, "year": 1951}, {"value": 0.2857142857142857, "year": 1952}, {"value": 0.4666666666666667, "year": 1953}, {"value": 0.1875, "year": 1954}, {"value": 0.35714285714285715, "year": 1955}, {"value": 0.35714285714285715, "year": 1956}, {"value": 0.2608695652173913, "year": 1957}, {"value": 0.35294117647058826, "year": 1958}, {"value": 0.6875, "year": 1959}, {"value": 0.4444444444444444, "year": 1960}, {"value": 0.26666666666666666, "year": 1961}, {"value": 0.5263157894736842, "year": 1962}, {"value": 0.2, "year": 1963}, {"value": 0.16666666666666666, "year": 1964}, {"value": 0.5, "year": 1965}, {"value": 0.3181818181818182, "year": 1966}, {"value": 0.5, "year": 1967}, {"value": 0.34782608695652173, "year": 1968}, {"value": 0.29411764705882354, "year": 1969}, {"value": 0.5, "year": 1970}, {"value": 0.38095238095238093, "year": 1971}, {"value": 0.47058823529411764, "year": 1972}, {"value": 0.3888888888888889, "year": 1973}, {"value": 0.38461538461538464, "year": 1974}, {"value": 0.3888888888888889, "year": 1975}, {"value": 0.4, "year": 1976}, {"value": 0.5, "year": 1977}, {"value": 0.3684210526315789, "year": 1978}, {"value": 0.16666666666666666, "year": 1979}, {"value": 0.4166666666666667, "year": 1980}, {"value": 0.3333333333333333, "year": 1981}, {"value": 0.39285714285714285, "year": 1982}, {"value": 0.47058823529411764, "year": 1983}, {"value": 0.4230769230769231, "year": 1984}, {"value": 0.5121951219512195, "year": 1985}, {"value": 0.375, "year": 1986}, {"value": 0.5116279069767442, "year": 1987}, {"value": 0.5641025641025641, "year": 1988}, {"value": 0.43243243243243246, "year": 1989}, {"value": 0.5142857142857142, "year": 1990}, {"value": 0.46511627906976744, "year": 1991}, {"value": 0.4473684210526316, "year": 1992}, {"value": 0.5, "year": 1993}, {"value": 0.4745762711864407, "year": 1994}, {"value": 0.47761194029850745, "year": 1995}, {"value": 0.5373134328358209, "year": 1996}, {"value": 0.5303030303030303, "year": 1997}, {"value": 0.42857142857142855, "year": 1998}, {"value": 0.42857142857142855, "year": 1999}, {"value": 0.4875, "year": 2000}, {"value": 0.5595238095238095, "year": 2001}, {"value": 0.5, "year": 2002}, {"value": 0.5444444444444444, "year": 2003}, {"value": 0.5446428571428571, "year": 2004}, {"value": 0.5350877192982456, "year": 2005}, {"value": 0.5352112676056338, "year": 2006}, {"value": 0.546583850931677, "year": 2007}, {"value": 0.5412371134020618, "year": 2008}, {"value": 0.4574898785425101, "year": 2009}, {"value": 0.458955223880597, "year": 2010}, {"value": 0.47104247104247104, "year": 2011}, {"value": 0.46551724137931033, "year": 2012}, {"value": 0.5, "year": 2013}]
#!/usr/bin/env python
"""
Really nasty little script to scrape the data off of bechdeltest.com
"""
import json
import requests
from bs4 import BeautifulSoup, Tag
resp = requests.get('http://bechdeltest.com/', params={'list': 'all'})
soup = BeautifulSoup(resp.content)
movie_list = soup.find(class_='list')
years = {}
year = None
for child in movie_list:
if not isinstance(child, Tag):
continue # ignore text space
try:
assert child.a['id'].startswith('year-')
except (AssertionError, TypeError, KeyError):
pass
else:
year = int(child.a['id'][len('year-'):])
years[year] = [0, 0] #pass/fail
try:
assert 'movie' in child['class']
except (AssertionError, KeyError):
pass
else:
years[year][int(child.img['src'] != '/static/ok.png')] += 1
for year in years:
y, n = years[year]
years[year] = y/float(y+n)
with open('data.json', 'w') as out:
json.dump([{'year': y, 'value': v} for y,v in years.items()], out)
<!DOCTYPE html>
<html>
<meta charset="utf-8">
<head>
<title></title>
<style>
.chart rect {
stroke: white;
fill: steelblue;
}
.chart text {
font: 10px sans-serif;
}
</style>
<script src="http://d3js.org/d3.v3.min.js"></script>
</head>
<body>
<script>
var h = 500 - 40,
w = 900 - 40;
var y = d3.scale.linear()
.domain([0, 1])
.range([0, h-20]);
var x = d3.scale.linear()
.domain([1900, 2013])
.range([0, w]);
var chart = d3.select("body").append("svg")
.attr("class", "chart")
.attr("height", h + 40)
.attr("width", w + 40)
.append('g')
.attr('transform', 'translate(20, 20)');
chart.selectAll('.tickline')
.data(y.ticks(10))
.enter().append('line')
.attr('class', 'tickline')
.attr('x1', 0)
.attr('x2', w)
.attr('y1', function(d) { return h - y(d); })
.attr('y2', function(d) { return h - y(d); })
.style("stroke-width", 1)
.style("stroke", "#ccc");
chart.selectAll('.percent')
.data(y.ticks(10))
.enter().append('text')
.attr('class', 'percent')
.attr('x', 20)
.attr('y', function(d) { return h - y(d); })
.text(String);
d3.json('data.json', function(error, data){
chart.selectAll("rect")
.data(data)
.enter().append("rect")
.attr("x", function(d) { return x(d.year); })
.attr('y', function(d) { return h - y(d.value); })
.attr('height', function(d) { return y(d.value); })
.attr("width", '4');
chart.append('line')
.attr('x1', 0)
.attr('x2', w)
.attr('y1', h)
.attr('y2', h)
.style("stroke-width", 2)
.style("stroke", "#000");
chart.selectAll('.rule')
.data(x.ticks(10))
.enter().append('text')
.attr('class', 'rule')
.attr('x', x)
.attr('y', h)
.attr('dy', 20)
.attr('text-anchor', 'middle')
.text(String)
})
</script>
</body>
</html>
beautifulsoup4
lxml
requests
@dschep

This comment has been minimized.

Copy link
Owner Author

dschep commented Mar 9, 2013

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.