moatbot_experiment.py

from pyramid.paster import bootstrap
env = bootstrap('moatbot_experiment.ini')

from datetime import datetime, timedelta
from scrapepipeline import US_DESKTOP_REGION, UK_DESKTOP_REGION
from scrapepipeline.potato2.client import GlobalEntity, TagGroup, Tag

from scrapepipeline.cache import ghostery

from itertools import chain, product
import json

bugs       = json.load(open('moat_ghostery_bugs/moat_ghostery_bugs.json'))
categories = ghostery.get_ghostery_categories('dev_ghostery_categories')

def creatives_count(region):
    ge = GlobalEntity(region=region)

    return ge.get_creatives().count()

def get_all_categories():
    return categories.keys()

def get_tags_by_categories(region, category_names):
    names = [ name for name, _ in chain(*[categories[cname] for cname in category_names]) ]
    aids = [bug['aid'] for bug in bugs['bugs'] if bug['name'] in names]

    tags = TagGroup([Tag(aid).region(region) for aid in aids])

    ge = GlobalEntity(region=region)

    stats = ge.get_impression_stats(filter_with=tags)[0]

    return stats.num_impressions

def main(regions):
    for region_name in regions.values():
        print region_name, creatives_count(region_name)

    category_names = get_all_categories()

    for region_name, category_name in product(regions.values(), category_names):
        print region_name, category_name, get_tags_by_categories(region_name, [category_name])

if __name__ == '__main__':
    regions = {
        'US': 'US',
        'UK': 'UK'
    }

    main(regions)