get the most important links tweeted during #openvisconf 2017

README.md

See twitter

atop_links.csv

neilk_edges.csv

parse.py

"""
Data courtesy of @estebanpdl from this tweet:
  https://twitter.com/estebanpdl/status/857278429774979073
data actually here:
    https://drive.google.com/file/d/0B5BuQR_OGa_WQ2F6Rm5hLUFPcDQ/view
"""

from collections import defaultdict
from multiprocessing import Pool, TimeoutError
from multiprocessing.pool import ThreadPool

import csv
import json
import os
import requests
import unicodecsv


def unshorten_url(url):
    """unshorten the given shortened url to see where it goes."""
    print(url)
    try:
        return requests.head(url, allow_redirects=True).url
    except Exception as e:
        print("Error retrieving %s: %s" % (url, e))
        return url


def read(filename='openvisconf2017_tweets.csv'):
    """Reading excel-exported csvs is not fun: http://stackoverflow.com/questions/33785422"""
    with open(filename, 'rbU') as f:
        r = unicodecsv.DictReader(f, encoding='utf-8-sig', dialect='excel')
        return list(r)


def expand_urls(data, filename):
    """Expands and caches urls; reads from cache if already completed."""
    if os.path.exists(filename):
        with open(filename) as f:
            return json.load(f)
    else:
        urls = [d['Urls'] for d in data if d['Urls'] != 'Null']
        to_expand = set(urls)
        pool = Pool(processes=8) # takes ~ 2 minutes
        expanded = pool.map(unshorten_url, to_expand)
        d = dict(zip(to_expand, expanded))

        # save expanded urls so we don't have to do this repeatedly.
        with open(filename, 'w') as f:
            json.dump(d, f, sort_keys=True, indent=4)

        return d


def write_updates(data, filename):
    """Write csv with additional fields"""
    fieldnames = data[0].keys()
    with open(filename, 'w') as f:
        w = unicodecsv.DictWriter(f, fieldnames)
        w.writeheader()
        w.writerows(data)


def fix(d):
    """type coercion for row dicts loaded from csv."""
    identity = lambda x: x
    rewriters = {'Favs': int, 'Rts': int}
    return {k: rewriters.get(k, identity)(v) for (k, v) in d.iteritems()}


def read_final(filename):
    """Read our nicely fixed up data."""
    if os.path.exists(filename):
        with open(filename) as f:
            r = unicodecsv.DictReader(f)
            return map(fix, r)
    else:
        # Make the data look the way we want it to.
        data = read()
        expanded = expand_urls(data, 'expanded.json')

        for d in data:
            d['url_expanded'] = expanded.get(d['Urls'])
        write_updates(data, filename)
        return map(fix, data)


def main():
    data = read_final('tweets.csv')
    scores = defaultdict(int)
    for d in data:
        url = d['url_expanded']
        if len(url):
            scores[url] += 1 + d['Rts']

    scored = sorted(scores.items(), key=lambda(k,v):(v,k), reverse=True)

    # Print links that were tweeted more than 10x (arbitrary threshold ftw!)
    most_tweeted_links = [(k,v) for (k, v) in scored if v > 10]
    with open('atop_links.csv', 'w') as f:
        w = csv.writer(f)
        w.writerow(["Count", "URL"])
        for url, count in most_tweeted_links:
            w.writerow([count, url])
            print("%d: %s" % (count, url))
    return scored


if __name__ == '__main__':
    main()