sync.py

#!/usr/bin/env python
'''
branches: eden, dharma, frodo-pre, eden-pre, dharma-pre, playground, master
'''
import os
import datetime
from git import Repo
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from boto.s3.bucket import Bucket
from mongokit import Connection
from xam import Repository, repos
from xab.models import Addon
from xab import config

#REPO_DIR='/Users/jbeluch/Repositories/xab/'
#REPO_NAMES = ['plugins', 'scrapers', 'screensavers', 'scripts', 'skins',
#REPO_NAMES = ['plugins', 'scrapers', 'scripts', 'skins',
              #'visualizations', 'webinterfaces']
              #'webinterfaces']

s3conn = S3Connection(config.AWS_ACCESS_KEY_ID, config.AWS_SECRET_ACCESS_KEY)
bucket = Bucket(s3conn, 'xbmcaddonbrowser.com')
bucket.set_acl('public-read')

connection = Connection('localhost', 27017)
connection.register([Addon])

def update_repos(repo_paths):
    # pull all the repos
    # upload icon images to s3
    # update the mongo document with an icon_image_url
    repos = [Repo(path) for path in repo_paths]
    branches = ['dharma', 'eden']
    for repo in repos:
        # Fetch remote changes
        repo.remotes.origin.fetch()

        # Update icon, fanart
        update_icon(repo, branches)


def calculate_etag(file):
    hasher = getattr(hashlib,'md5')()
    data = file.read(1024*64)
    while data:
        hasher.update(data)
        data = file.read(1024*64)
    return hasher.hexdigest()


def update_icon(repo, branches):
    print repo, repo.refs
    for branch in branches:
        # Checkout the local branch
        try:
            repo.refs['origin/' + branch].checkout()
        except TypeError:
            pass  # shouldn't be thrown

        wdir = repo.working_dir
        for addon_id in os.listdir(wdir):
            icon = os.path.join(wdir, addon_id, 'icon.png')
            if os.path.exists(icon):
                # attempt to upload to s3
                new_url = upload_to_storage(icon, addon_id, branch)

                if new_url:
                    #set in mongo
                    addon = connection.xabtest2.addons.update({'addon_id': addon_id, 'repository': branch}, {'$set': {'icon_url': new_url}})

            fanart = os.path.join(wdir, addon_id, 'fanart.jpg')
            if os.path.exists(fanart):
                # attempt to upload to s3
                new_url = upload_to_storage(fanart, addon_id, branch, '%s_%s_fanart.jpg' % (addon_id, branch))

                if new_url:
                    #set in mongo
                    addon = connection.xabtest2.addons.update({'addon_id': addon_id, 'repository': branch}, {'$set': {'fanart_url': new_url}})


def upload_to_storage(filename, addon_id, branch, dst_filename=None):
    if dst_filename:
        key_name = dst_filename
    else:
        key_name = '%s_%s.png' % (addon_id, branch)

    existing_key = bucket.get_key(key_name)
    if existing_key:
        remote_etag = existing_key.etag.strip('"')
        local_etag = existing_key.compute_md5(open(filename))[0]
        if remote_etag == local_etag:
            print 'Skipping %s...' % filename
            return 'https://s3.amazonaws.com/xbmcaddonbrowser.com/%s' % key_name
            #return

    print 'Uploading %s...' % filename
    key = Key(bucket)
    key.key = key_name
    key.set_contents_from_filename(filename, policy='public-read')
    return 'https://s3.amazonaws.com/xbmcaddonbrowser.com/%s' % key_name



# Load up the repos
def clone_repos(parent_dir, repo_names):
    for repo_name in repo_names:
		repo_dir = os.path.join(parent_dir, repo_name)
		if not os.path.isdir(repo_dir):
			print '* Cloning %s repo to %s' % (repo_name, repo_dir)
			print '* This may take a few minutes...'
			Repo.clone_from('git://xbmc.git.sourceforge.net/gitroot/xbmc/%s' % repo_name, repo_dir)

    # TODO: clone all the proper branches

def update_addons(addons, repo_name):
    for addon in addons:
        info = Addon.dict_from_xab_addon(addon)
        now = datetime.datetime.utcnow()
        info.update({'date_updated': now})
            
        item = connection.xabtest2.addons.find_one({'addon_id': addon.id, 'repository': repo_name})
        if item is not None:
            connection.xabtest2.addons.update({'addon_id': addon.id, 'repository': repo_name}, {'$set': info}, upsert=True, safe=True)
        else:
            # must create a new entry
            doc = connection.Addon()
            doc.update(info)
            doc['repository'] = unicode(repo_name)
            doc.save()

def update_from_xml():
    for repo_name in ['EDEN', 'DHARMA']:
        repo = Repository(*getattr(repos, repo_name))
        update_addons(repo.addons, repo_name.lower())

def main():
    clone_repos(config.GIT_PARENT_DIR, config.GIT_REPO_NAMES)
    update_from_xml()
    update_repos([os.path.join(config.GIT_PARENT_DIR, name) for name in config.GIT_REPO_NAMES])

if __name__ == '__main__':
    main()