Converts a directory of Python packages installed by pip into importable zip files for use on Google App Engine.

zipimportify.py

#!/usr/bin/env python
"""
Convert a directory of pip-installed libraries into zip-imports for use on GAE.

Usage:

    pip install -U <library_name>
    pip freeze > requirements_dev.txt
    pip install -U --egg --target <path_to_zips> -r requirements_dev.txt
    python zipimportify.py <path_to_zips>
    cd <path_to_zips>
    ls | grep -v '\.zip' | xargs rm -rf
"""
import argparse
import json
import os
import subprocess
import sys


PIP_INFO_SUFFIX = '-info'
TOP_LEVEL = 'top_level.txt'
METADATA = 'metadata.json'

parser = argparse.ArgumentParser(description='Zip up and version pip-installed libraries.')
parser.add_argument('path', type=str)


def top_level(path):
  with open(os.path.join(path, TOP_LEVEL), 'r') as fp:
    line = fp.readline().strip()
    # Note(eric): Packages like PyYaml have top_level lines like "__yaml" that should be skipped.
    while line.startswith('_'):
      line = fp.readline().strip()
    return line


def version(path):
  with open(os.path.join(path, METADATA), 'r') as fp:
    metadata = json.load(fp)
    return metadata['version']


def zip_dir(dir_to_zip, filename, cwd=None):
  return subprocess.Popen(['zip', '-qr0', filename, dir_to_zip], cwd=cwd, universal_newlines=True)

def zip_files(files_to_zip, filename, cwd=None):
  return subprocess.Popen(['zip', '-qr0', filename, '.', '-i', files_to_zip], cwd=cwd, universal_newlines=True)


def main(args):
  # Everything in the directory.
  all_files = set(os.listdir(args.path))
  # Everything that ends with "-info".
  info_names = set([x for x in all_files if x.endswith(PIP_INFO_SUFFIX)])
  print len(info_names), '-info files:'
  print info_names
  print

  # Create paths by prepending the directory.
  info_paths = set([os.path.join(args.path, x) for x in info_names])
  print len(info_paths), 'info paths:'
  print info_paths
  print

  # Every package version that matches a subdirectory name.
  versions = {top_level(x): version(x) for x in info_paths}
  print len(versions.keys()), 'versions:'
  print versions
  print

  packages = set([x for x in versions.iterkeys() if os.path.isdir(os.path.join(args.path, x))])
  print len(packages), 'packages:'
  print packages
  print

  non_dir_packages = set(versions.iterkeys()) - packages
  print len(non_dir_packages), 'non_dir_packages:'
  print non_dir_packages
  print

  # Create a zip for each package.
  print 'zip paths:'
  for path in packages:
    zip_path = path + '-' + versions[path] + '.zip'
    print path, '>', zip_path
    zip_dir(path, zip_path, cwd=args.path)
  print

  # Create a zip for each non-directory package.
  print 'zip paths:'
  for path in non_dir_packages:
    zip_path = path + '-' + versions[path] + '.zip'
    path = path + '.py*'
    print path, '>', zip_path
    zip_files(path, zip_path, cwd=args.path)
  print


if __name__ == '__main__':
  args = parser.parse_args()
  main(args)