Skip to content

Instantly share code, notes, and snippets.

@jbeluch
Created August 29, 2012 22:08
Show Gist options
  • Save jbeluch/3519522 to your computer and use it in GitHub Desktop.
Save jbeluch/3519522 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python
'''
branches: eden, dharma, frodo-pre, eden-pre, dharma-pre, playground, master
'''
import os
import datetime
from git import Repo
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from boto.s3.bucket import Bucket
from mongokit import Connection
from xam import Repository, repos
from xab.models import Addon
from xab import config
#REPO_DIR='/Users/jbeluch/Repositories/xab/'
#REPO_NAMES = ['plugins', 'scrapers', 'screensavers', 'scripts', 'skins',
#REPO_NAMES = ['plugins', 'scrapers', 'scripts', 'skins',
#'visualizations', 'webinterfaces']
#'webinterfaces']
s3conn = S3Connection(config.AWS_ACCESS_KEY_ID, config.AWS_SECRET_ACCESS_KEY)
bucket = Bucket(s3conn, 'xbmcaddonbrowser.com')
bucket.set_acl('public-read')
connection = Connection('localhost', 27017)
connection.register([Addon])
def update_repos(repo_paths):
# pull all the repos
# upload icon images to s3
# update the mongo document with an icon_image_url
repos = [Repo(path) for path in repo_paths]
branches = ['dharma', 'eden']
for repo in repos:
# Fetch remote changes
repo.remotes.origin.fetch()
# Update icon, fanart
update_icon(repo, branches)
def calculate_etag(file):
hasher = getattr(hashlib,'md5')()
data = file.read(1024*64)
while data:
hasher.update(data)
data = file.read(1024*64)
return hasher.hexdigest()
def update_icon(repo, branches):
print repo, repo.refs
for branch in branches:
# Checkout the local branch
try:
repo.refs['origin/' + branch].checkout()
except TypeError:
pass # shouldn't be thrown
wdir = repo.working_dir
for addon_id in os.listdir(wdir):
icon = os.path.join(wdir, addon_id, 'icon.png')
if os.path.exists(icon):
# attempt to upload to s3
new_url = upload_to_storage(icon, addon_id, branch)
if new_url:
#set in mongo
addon = connection.xabtest2.addons.update({'addon_id': addon_id, 'repository': branch}, {'$set': {'icon_url': new_url}})
fanart = os.path.join(wdir, addon_id, 'fanart.jpg')
if os.path.exists(fanart):
# attempt to upload to s3
new_url = upload_to_storage(fanart, addon_id, branch, '%s_%s_fanart.jpg' % (addon_id, branch))
if new_url:
#set in mongo
addon = connection.xabtest2.addons.update({'addon_id': addon_id, 'repository': branch}, {'$set': {'fanart_url': new_url}})
def upload_to_storage(filename, addon_id, branch, dst_filename=None):
if dst_filename:
key_name = dst_filename
else:
key_name = '%s_%s.png' % (addon_id, branch)
existing_key = bucket.get_key(key_name)
if existing_key:
remote_etag = existing_key.etag.strip('"')
local_etag = existing_key.compute_md5(open(filename))[0]
if remote_etag == local_etag:
print 'Skipping %s...' % filename
return 'https://s3.amazonaws.com/xbmcaddonbrowser.com/%s' % key_name
#return
print 'Uploading %s...' % filename
key = Key(bucket)
key.key = key_name
key.set_contents_from_filename(filename, policy='public-read')
return 'https://s3.amazonaws.com/xbmcaddonbrowser.com/%s' % key_name
# Load up the repos
def clone_repos(parent_dir, repo_names):
for repo_name in repo_names:
repo_dir = os.path.join(parent_dir, repo_name)
if not os.path.isdir(repo_dir):
print '* Cloning %s repo to %s' % (repo_name, repo_dir)
print '* This may take a few minutes...'
Repo.clone_from('git://xbmc.git.sourceforge.net/gitroot/xbmc/%s' % repo_name, repo_dir)
# TODO: clone all the proper branches
def update_addons(addons, repo_name):
for addon in addons:
info = Addon.dict_from_xab_addon(addon)
now = datetime.datetime.utcnow()
info.update({'date_updated': now})
item = connection.xabtest2.addons.find_one({'addon_id': addon.id, 'repository': repo_name})
if item is not None:
connection.xabtest2.addons.update({'addon_id': addon.id, 'repository': repo_name}, {'$set': info}, upsert=True, safe=True)
else:
# must create a new entry
doc = connection.Addon()
doc.update(info)
doc['repository'] = unicode(repo_name)
doc.save()
def update_from_xml():
for repo_name in ['EDEN', 'DHARMA']:
repo = Repository(*getattr(repos, repo_name))
update_addons(repo.addons, repo_name.lower())
def main():
clone_repos(config.GIT_PARENT_DIR, config.GIT_REPO_NAMES)
update_from_xml()
update_repos([os.path.join(config.GIT_PARENT_DIR, name) for name in config.GIT_REPO_NAMES])
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment