EMPMYMIP: Easily Maintain a PyPI Mirror of Your Important Packages!

README.rst

EMPMYMIP: Easily Maintain a PyPI Mirror of Your Important Packages!

Status

EMPMYMIP seems to mostly kind of work for me. It might even do the same for you!

The implementation is kind of a huge hack, though, and it might very well explode if pip changes the behaviour of their bundle command.

Instructions

0. Create a pypi directory somewhere under htdocs:

   $ mkdir /var/www/pypi/

1. Use pip2tgz to download all your requirements:

   $ pip2tgz /var/www/pypi/ -r project/requirements.txt some-package==1.2

2. Use dir2pypi to build a pypi-compatible "simple" index:

   $ dir2pypi /var/www/pypi/

3. Prefix your requirements.txt with --use-index=http://example.com/pypi/simple. From now on, pip install -r requirements.txt will download packages from your local mirror. New packages can be added by repeating steps 2 and 3.

dijkstra?

I've included the pip2dijkstra and pkg2dijkstra scripts that we use in-house to automatically push new packages to our development server. When I add a new requirement (or change a version of a requirement), I can run run pip2dijkstra new-package==1.2 to pull in the new-package-1.2.tar.gz and publish it, all in one command (or I can use pkg2dijkstra new-package-1.2.tar.gz to publish a package without downloading it from PyPI).

dir2pypi

#!/usr/bin/env python

import os
import sys
import shutil

def file_to_package(file):
    """ Returns the package name for a given file.
        >>> file_to_package("foo-1.2.3_rc1.tar.gz")
        ('foo', '1.2.3-rc1.tar.gz')
        >>> file_to_package("foo-bar-1.2.tgz")
        ('foo-bar', '1.2.tgz')
        >>> """
    split = file.rsplit("-", 1)
    if len(split) != 2:
        msg = ("unexpected file name: %r (not in "
               "'package-name-version.xxx' format)") %(file, )
        raise ValueError(msg)
    return (split[0], split[1].replace("_", "-"))

def main():
    if len(sys.argv) != 2:
        print "usage: %s PACKAGE_DIR"
        print "Creates the directory PACKAGE_DIR/simple/ and populates it"
        print "with the directory structure required to use with pip's"
        print "--index-url"
        print
        print "Assumes that PACKAGE_DIR contains a bunch of archives"
        print "named 'package-name-version.ext' (ex 'foo-2.1.tar.gz'"
        print "or 'foo-bar-1.3rc1.bz2')"
        print
        print "This makes the most sense if PACKAGE_DIR is somewhere"
        print "inside a webserver's inside htdocs directory."
        sys.exit(1)
    pkgdir = sys.argv[1]
    if not os.path.isdir(pkgdir):
        raise ValueError("no such directory: %r" %(pkgdir, ))
    pkgdirpath = lambda *x: os.path.join(pkgdir, *x)

    shutil.rmtree(pkgdirpath("simple"), ignore_errors=True)
    os.mkdir(pkgdirpath("simple"))

    for file in os.listdir(pkgdir):
        pkg_filepath = os.path.join(pkgdir, file)
        if not os.path.isfile(pkg_filepath):
            continue
        pkg_basename = os.path.basename(file)
        pkg_name, pkg_rest = file_to_package(pkg_basename)
        pkg_dir = pkgdirpath("simple", pkg_name)
        if not os.path.exists(pkg_dir):
            os.mkdir(pkg_dir)
        pkg_new_basename = "-".join([pkg_name, pkg_rest])
        os.link(pkg_filepath, os.path.join(pkg_dir, pkg_new_basename))

if __name__ == "__main__":
    main()

pip2dijkstra

#!/bin/bash

IFS="`printf "\n\t"`"
set -eu

if [[ "$#" -lt 1 ]]; then
    echo "usage: $0 PACKAGE_NAME ..."
    exit 1
fi

pip2tgz="$(dirname "$0")/pip2tgz"
pkg2dijkstra="$(dirname "$0")/pkg2dijkstra"

tempdir="/tmp/_pip2dijkstra_temp/"
mkdir "$tempdir" 2>&1 > /dev/null

$pip2tgz "$tempdir" "$@"
files=($tempdir/*)
$pkg2dijkstra "${files[@]}"
rm -r "$tempdir"

pip2tgz

#!/usr/bin/env python

import os
import sys
import shutil
from subprocess import check_call

if len(sys.argv) < 3:
    import textwrap
    print textwrap.dedent("""
        usage: %s OUTPUT_DIRECTORY PACKAGE_NAMES

        Where PACKAGE_NAMES are any names accepted by pip (ex, `foo`,
        `foo==1.2`, `-r requirements.txt`).
        
        %s will download all packages required to install PACKAGE_NAMES and
        save them to sanely-named tarballs in OUTPUT_DIRECTORY
    """ %(sys.argv[0], sys.argv[0]))
    sys.exit(1)

outdir = os.path.abspath(sys.argv[1])

tempdir = os.path.join(outdir, "_pip2tgz_temp")
if os.path.exists(tempdir):
    shutil.rmtree(tempdir)
os.mkdir(tempdir)

bundle_zip = os.path.join(tempdir, "bundle.zip")
check_call(["pip", "bundle", "-b", tempdir, bundle_zip] + sys.argv[2:])

os.chdir(tempdir)
check_call(["unzip", "bundle.zip", "pip-manifest.txt"])

for line in open("pip-manifest.txt"):
    line = line.strip()
    if not line or line.startswith("#"):
        continue
    pkg_version = line.split("==")
    if len(pkg_version) != 2:
        raise ValueError("surprising line: %r" %(line, ))
    pkg, version = pkg_version
    version = version.replace("-", "_")
    old_input_dir = pkg
    new_input_dir = "%s-%s" %(pkg, version)
    os.rename(old_input_dir, new_input_dir)
    output_name = os.path.join("..", new_input_dir + ".tar.gz")
    check_call(["tar", "-czvf", output_name, new_input_dir])

os.chdir(outdir)
shutil.rmtree(tempdir)

pkg2dijkstra

#!/bin/bash

IFS="`printf "\n\t"`"
set -eu

if [[ -z "${1-}" ]]; then
    echo "usage: $0 (foo-1.2.3.tar.gz|foo/) ..."
    exit 1
fi

targets=("$@")
for (( i = 0; i < $#; i += 1)); do
    target="${targets[$i]}"
    if [[ -d "$target" ]]; then
        echo "looks like a source directory. Doing some magic to make that work..."
        pushd "$target" > /dev/null
        rm -r dist/ 2>&1 > /dev/null || true
        python setup.py sdist
        target="$(echo "$PWD/dist/"*)"
        popd > /dev/null
        targets[$i]="$target"
    fi
done

scp "${targets[@]}" luminaut@dijkstra.local:/var/www/pypi/

ssh luminaut@dijkstra.local "cd /var/www/; ./dir2pypi pypi"