Skip to content

Instantly share code, notes, and snippets.

@sheymann

sheymann/entries.md

Forked from samuelclay/entries.md
Created June 7, 2012 15:51
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sheymann/2889596 to your computer and use it in GitHub Desktop.
Save sheymann/2889596 to your computer and use it in GitHub Desktop.
Download ZIP
Top entries in Knight News Challenge 2012
Raw
entries.md

Semi-Finalists:

Found 425 active entries among 961 total applications.

57 semi-finalists, 41 with at least 1 like.

Bold denotes semi-finalist, moving on to the finals
Raw
newschallenge.py
# Screen scrapes the Knight News Challenge entries (all 64 pages of them)
# and counts the number of votes/hearts for each entry. Then displays them
# in rank order.
#
# This script runs in about 20 seconds.
import requests
from BeautifulSoup import BeautifulSoup
# Winners found on http://newschallenge.tumblr.com/post/20962258701/knight-news-challenge-on-networks-moving-to-the-next:
#
# $('.posts .MsoNormal > span').find('a[href^="http://newschallenge.tumblr.com/post"]').map(function() {
# return $(this).attr('href');
# });
winners = [
"http://newschallenge.tumblr.com/post/20962258701/knight-news-challenge-on-networks-moving-to-the-next#disqus_thread",
"http://newschallenge.tumblr.com/post/19436493313/amauta-a-collaborative-media-network",
"http://newschallenge.tumblr.com/post/19493987224/cont3nt-com-lets-you-sell-media-in-real-time-via",
"http://newschallenge.tumblr.com/post/19494011127/expand-the-unconsumption-project",
"http://newschallenge.tumblr.com/post/19493557384/mediareputations-com-verifies-your-credentials-and",
"http://newschallenge.tumblr.com/post/19438230966/prescouter-storify-meets-wikipedia",
"http://newschallenge.tumblr.com/post/20968613548/themes-surprises-and-outliers-from-1000",
"http://newschallenge.tumblr.com/post/19436493313/amauta-a-collaborative-media-network",
"http://newschallenge.tumblr.com/post/19436676620/filling-foreign-news-gaps-with-scholars-asia-beat",
"http://newschallenge.tumblr.com/post/19478851354/bridging-the-big-data-digital-divide-information",
"http://newschallenge.tumblr.com/post/19492881188/1-what-do-you-propose-to-do-20-words-scale",
"http://newschallenge.tumblr.com/post/19121005017/citjo-connecting-twitter-users-with-media-buyers",
"http://newschallenge.tumblr.com/post/19479493999/connecting-the-global-hacks-hackers-network",
"http://newschallenge.tumblr.com/post/19436607188/connecting-the-world-with-rural-india-via-facebook-and",
"http://newschallenge.tumblr.com/post/19493987224/cont3nt-com-lets-you-sell-media-in-real-time-via",
"http://newschallenge.tumblr.com/post/19438970667/the-cowbird-community-reporting-project",
"http://newschallenge.tumblr.com/post/19450699629/new-contribution-tools-for-openstreetmap",
"http://newschallenge.tumblr.com/post/19479653130/differentfeather",
"http://newschallenge.tumblr.com/post/19478834324/diy-drone-fleets-for-airborne-web-journalism",
"http://newschallenge.tumblr.com/post/19483270689/docs-to-wordpress-to-indesign",
"http://newschallenge.tumblr.com/post/19477903682/electoral-college-of-me",
"http://newschallenge.tumblr.com/post/19404846313/envirofact",
"http://newschallenge.tumblr.com/post/19490695157/funf-org-open-mobile-sensing",
"http://newschallenge.tumblr.com/post/19490695157/funf-org-open-mobile-sensing",
"http://newschallenge.tumblr.com/post/19419901491/global-censorship-monitoring-system",
"http://newschallenge.tumblr.com/post/19065611908/a-google-news-for-the-social-web",
"http://newschallenge.tumblr.com/post/19438785842/hawaii-eco-net",
"http://newschallenge.tumblr.com/post/19180046026/hypothes-is-an-annotation-layer-for-the-web",
"http://newschallenge.tumblr.com/post/19479029243/iava-new-gi-bill-veterans-alumni-network-vets",
"http://newschallenge.tumblr.com/post/19436574450/m-health-news-network",
"http://newschallenge.tumblr.com/post/19493557384/mediareputations-com-verifies-your-credentials-and",
"http://newschallenge.tumblr.com/post/19480304461/mesh-potato-2-0",
"http://newschallenge.tumblr.com/post/19479664924/mobile-publishing-for-everyone",
"http://newschallenge.tumblr.com/post/19494194541/noula-crowdsourcing-needs-mapping-and-developping",
"http://newschallenge.tumblr.com/post/19484970513/peepol-tv-live-tv-powered-by-and-for-the-people",
"http://newschallenge.tumblr.com/post/19438230966/prescouter-storify-meets-wikipedia",
"http://newschallenge.tumblr.com/post/19479504346/prozr-twitter-stories-in-a-snap",
"http://newschallenge.tumblr.com/post/19345456890/rbutr-follow-online-discourse-between-websites",
"http://newschallenge.tumblr.com/post/18794349346/recovers-org-community-powered-disaster-recovery",
"http://newschallenge.tumblr.com/post/19436424823/secure-anonymous-journalism-toolkit",
"http://newschallenge.tumblr.com/post/19021661497/sensor-networks-for-news",
"http://newschallenge.tumblr.com/post/19450685278/tethr-evolving-networks",
"http://newschallenge.tumblr.com/post/19293910540/the-pressforward-dashboard",
"http://newschallenge.tumblr.com/post/18576274733/thinkup",
"http://newschallenge.tumblr.com/post/19345435254/1-what-do-you-propose-to-do-20-words-build-a",
"http://newschallenge.tumblr.com/post/19490689958/truth-goggles",
"http://newschallenge.tumblr.com/post/19403515934/truth-teller",
"http://newschallenge.tumblr.com/post/19494011127/expand-the-unconsumption-project",
"http://newschallenge.tumblr.com/post/19290074949/unicef-gis-youth-led-digital-mapping",
"http://newschallenge.tumblr.com/post/19397319461/watchup-the-first-news-watcher",
"http://newschallenge.tumblr.com/post/19493588407/water-canary",
"http://newschallenge.tumblr.com/post/19480319147/a-bridge-between-wordpress-and-git",
"http://newschallenge.tumblr.com/post/19414762330/in-the-life-media-transforming-lgbt-journalism",
"http://newschallenge.tumblr.com/post/19493920734/get-to-the-source",
"http://newschallenge.tumblr.com/post/19480128205/farm-to-table-school-lunch",
"http://newschallenge.tumblr.com/post/19477700441/partisans-org",
"http://newschallenge.tumblr.com/post/19345505702/protecting-journalists-and-engaging-communities"]
def find_entries():
page = 1
total_entry_count = 0
entries = []
while True:
print " ---> Found %s entries so far. Now on page: %s" % (len(entries), page)
knight_url = "http://newschallenge.tumblr.com/page/%s" % (page)
html = requests.get(knight_url).content
soup = BeautifulSoup(html)
postboxes = soup.findAll("div", "postbox")
# Done if only sticky entry is left.
if len(postboxes) <= 1:
break
page += 1
# 15 entries per page, plus a sticky throwaway entry
for entry in postboxes:
if 'stickyPost' in entry.get('class'): continue
total_entry_count += 1
likes = entry.find("", "home-likes")
if likes and likes.text:
likes = int(likes.text)
else:
likes = 0
comments = entry.find("", "home-comments")
if comments and comments.text:
comments = int(comments.text)
else:
comments = 0
title = entry.find("h2")
if title:
title = title.text
url = entry.find('a', "home-view")
if url:
url = url.get('href')
# Only record active entries
if comments or likes:
entries.append({
'likes': likes,
'comments': comments,
'title': title,
'url': url,
})
# time.sleep(random.randint(0, 2))
entries.sort(key=lambda e: e['comments'] + e['likes'])
entries.reverse()
active_entry_count = len(entries)
found_entries = []
winner_count = 0
for i, entry in enumerate(entries):
is_winner = entry['url'] in winners
if is_winner: winner_count += 1
print " * %s#%s: %s likes - [%s](%s)%s" % (
"**" if is_winner else "",
i+1,
entry['likes'], entry['title'],
entry['url'],
"**" if is_winner else "")
found_entries.append(entry)
print " ***> Found %s active entries among %s total applications with %s/%s winners." % (
active_entry_count, total_entry_count, winner_count, len(winners))
return found_entries
if __name__ == '__main__':
find_entries()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment