Gemfury Download

get_gemfury_packages.py

#!/usr/bin/env python
# Copyright (C) Micah Hausler, 2014
# Using MIT License <http://opensource.org/licenses/MIT>
#

__doc__ = """
This is a quick hack to download all your python packages from Gemfury. 

$ virtualenv venv
$ source ./venv/bin/activate
$ python get_gemfury_packages.py <token> [organization]
$ ls ./gemfury 

"""

import sys
import os
import subprocess
import urllib2
import json


class Fury(object):
    """
    A class to interact with the Gemfury API
    """
    URL = 'https://api.fury.io/1'

    def __init__(self, authorization_token, DOWNLOAD_DIR=None, org=None):
        self.DOWNLOAD_DIR = DOWNLOAD_DIR or './gemfury/'
        self._TOKEN = authorization_token
        self._ORGANIZATION = org

        self.HEADERS = {
                'Content-Type': 'application/json',
                'Authorization': authorization_token,
                'User-Agent': 'Gemfury RubyGem 0.4.25',
        }

    def _get_endpoint(self, endpoint, params=None):
        url_template = '{url}/{endpoint}?{params}'

        if self._ORGANIZATION and not params:
            params = ['as={}'.format(self._ORGANIZATION)]
        elif not params:
            params = []

        url = url_template.format(
            url=self.URL,
            endpoint=endpoint,
            params='&'.join(params)
        )

        response = urllib2.urlopen(
            urllib2.Request(
                url,
                headers=self.HEADERS
            )
        )
        output = ''.join(response.readlines())

        return json.loads(output)

    def list_packages(self):
        return self._get_endpoint('gems/')

    def list_versions(self, package):
        if type(package) is dict:
            name = package.get('name')
        else:
            name = package

        return self._get_endpoint('gems/{name}/versions/'.format(name=name))

    def list_all_versions(self):
        packages = self.list_packages()

        data = {}
        for package in packages:
            data[package['name']] = self.list_versions(package)

        return data

    def get_all_version_strings(self):
        packages = self.list_all_versions()

        all_versions = []

        for package, versions in packages.iteritems():
           for version in versions:
              all_versions.append('{}=={}'.format(package, version.get('version')))

        return all_versions

    def download_all(self):

        os.system('mkdir -p {}'.format(self.DOWNLOAD_DIR))

        for package in self.get_all_version_strings():
            p_name, version = package.split('==')
            if not os.path.isfile('{}/{}-{}.tar.gz'.format(self.DOWNLOAD_DIR, p_name, version)):
                cmd = 'pip install -i https://pypi.fury.io/{token}/{org}/ {package} -d {download_dir} --no-deps'.format(
                    token=self._TOKEN,
                    org=self._ORGANIZATION,
                    package=package,
                    download_dir=self.DOWNLOAD_DIR
                )
                print(cmd)
                output = subprocess.check_output(cmd.split())
                print(output)

if __name__ == '__main__':
    if len(sys.argv) == 1:
        print('Usage:')
        print('    ./get_gemfury_packages.py <token> [organization]')
        exit(1)
    token = sys.argv[1]
    org = None
    if len(sys.argv) == 3:
        org = sys.argv[2]

    f = Fury(token, org=org)
    f.download_all()