Pyquery scraper for logolounge.com Logo Trends data.

logolounge-trends-scraper

from pyquery import PyQuery as pq
import json

base_url = 'http://www.logolounge.com/article/'


years = ['2003trends', '2004trends', '2005trends', '2006trends',
         '2007trends', '2008trends', 'minortrendanimotion',
         '2009trends', '2010trends', '2011trends',
         '2012logotrends', '2013logotrends', '2014logotrends']


def get_year_html(year):
    site = pq('{}{}'.format(
        base_url, year),
        headers={'user-agent': 'pyquery-scraper'})
    return pq(site).find('#single-article').html().encode('utf8').strip()


def get_all_years():
    for year in years:
        print 'Writing file... {}'.format(year)
        f = open(year + '.html', 'wb')
        f.write(get_year_html(year))
        f.close()


def get_categories_per_page(page):
    return pq(filename=page).find(
        'h2:not([itemprop="headline"])').text().strip().split(' ')


def get_all_categories():
    all_cats = []
    for year in years:
        # Normalize data and create some categories
        # objects to work with as json
        cats = get_categories_per_page(year + '.html')
        obj = {'year': year.replace('logotrends', '').replace('trends', ''),
               'total': len(cats),
               'categories': cats}
        all_cats.append(obj)

    print 'Writing categories...'
    print all_cats
    f = open('all-categories.json', 'wb')
    f.write(json.dumps(all_cats) + '\n')
    f.close()


get_all_categories()

The idea is to grab the html first to minimize server requests. Then parse it locally and process as json.