Skip to content

Instantly share code, notes, and snippets.

@stevepiercy

stevepiercy/pyvenvex.py

Forked from vsajip/pyvenvex.py
Last active November 7, 2017 21:45
Download ZIP

Revisions

  1. stevepiercy revised this gist Jun 30, 2016. 1 changed file with 26 additions and 22 deletions.
    48 changes: 26 additions & 22 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -13,12 +13,12 @@
    class ExtendedEnvBuilder(venv.EnvBuilder):
    """
    This builder installs setuptools and pip so that you can pip or
    easy_install other packages into the created environment.
    easy_install other packages into the created virtual environment.
    :param nodist: If True, setuptools and pip are not installed into the
    created environment.
    created virtual environment.
    :param nopip: If True, pip is not installed into the created
    environment.
    virtual environment.
    :param progress: If setuptools or pip are installed, the progress of the
    installation can be monitored by passing a progress
    callable. If specified, it is called with two
    @@ -44,10 +44,10 @@ def __init__(self, *args, **kwargs):
    def post_setup(self, context):
    """
    Set up any packages which need to be pre-installed into the
    environment being created.
    virtual environment being created.
    :param context: The information for the environment creation request
    being processed.
    :param context: The information for the virtual environment
    creation request being processed.
    """
    os.environ['VIRTUAL_ENV'] = context.env_dir
    if not self.nodist:
    @@ -81,7 +81,7 @@ def install_script(self, context, name, url):
    fn = os.path.split(path)[-1]
    binpath = context.bin_path
    distpath = os.path.join(binpath, fn)
    # Download script into the env's binaries folder
    # Download script into the virtual environment's binaries folder
    urlretrieve(url, distpath)
    progress = self.progress
    if self.verbose:
    @@ -93,7 +93,7 @@ def install_script(self, context, name, url):
    else:
    sys.stderr.write('Installing %s ...%s' % (name, term))
    sys.stderr.flush()
    # Install in the env
    # Install in the virtual environment
    args = [context.env_exe, fn]
    p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath)
    t1 = Thread(target=self.reader, args=(p.stdout, 'stdout'))
    @@ -112,10 +112,10 @@ def install_script(self, context, name, url):

    def install_setuptools(self, context):
    """
    Install setuptools in the environment.
    Install setuptools in the virtual environment.
    :param context: The information for the environment creation request
    being processed.
    :param context: The information for the virtual environment
    creation request being processed.
    """
    url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py'
    self.install_script(context, 'setuptools', url)
    @@ -128,10 +128,10 @@ def install_setuptools(self, context):

    def install_pip(self, context):
    """
    Install pip in the environment.
    Install pip in the virtual environment.
    :param context: The information for the environment creation request
    being processed.
    :param context: The information for the virtual environment
    creation request being processed.
    """
    url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py'
    self.install_script(context, 'pip', url)
    @@ -154,7 +154,8 @@ def main(args=None):
    'more target '
    'directories.')
    parser.add_argument('dirs', metavar='ENV_DIR', nargs='+',
    help='A directory to create the environment in.')
    help='A directory in which to create the
    'virtual environment.')
    parser.add_argument('--no-setuptools', default=False,
    action='store_true', dest='nodist',
    help="Don't install setuptools or pip in the "
    @@ -178,14 +179,17 @@ def main(args=None):
    'the platform.')
    parser.add_argument('--clear', default=False, action='store_true',
    dest='clear', help='Delete the contents of the '
    'environment directory if it '
    'already exists, before '
    'virtual environment '
    'directory if it already '
    'exists, before virtual '
    'environment creation.')
    parser.add_argument('--upgrade', default=False, action='store_true',
    dest='upgrade', help='Upgrade the environment '
    'directory to use this version '
    'of Python, assuming Python '
    'has been upgraded in-place.')
    dest='upgrade', help='Upgrade the virtual '
    'environment directory to '
    'use this version of '
    'Python, assuming Python '
    'has been upgraded '
    'in-place.')
    parser.add_argument('--verbose', default=False, action='store_true',
    dest='verbose', help='Display the output '
    'from the scripts which '
    @@ -210,4 +214,4 @@ def main(args=None):
    rc = 0
    except Exception as e:
    print('Error: %s' % e, file=sys.stderr)
    sys.exit(rc)
    sys.exit(rc)
  2. @vsajip vsajip revised this gist Jul 12, 2013. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -74,6 +74,7 @@ def reader(self, stream, context):
    else:
    sys.stderr.write(s.decode('utf-8'))
    sys.stderr.flush()
    stream.close()

    def install_script(self, context, name, url):
    _, _, path, _, _, _ = urlparse(url)
  3. @vsajip vsajip revised this gist Jul 12, 2013. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -84,13 +84,13 @@ def install_script(self, context, name, url):
    urlretrieve(url, distpath)
    progress = self.progress
    if self.verbose:
    dots = '...\n'
    term = '\n'
    else:
    dots = '...'
    term = ''
    if progress is not None:
    progress('Installing %s %s' % (name, dots), 'main')
    progress('Installing %s ...%s' % (name, term), 'main')
    else:
    sys.stderr.write('Installing %s %s' % (name, dots))
    sys.stderr.write('Installing %s ...%s' % (name, term))
    sys.stderr.flush()
    # Install in the env
    args = [context.env_exe, fn]
  4. @vsajip vsajip revised this gist Jul 12, 2013. 1 changed file with 6 additions and 2 deletions.
    8 changes: 6 additions & 2 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -83,10 +83,14 @@ def install_script(self, context, name, url):
    # Download script into the env's binaries folder
    urlretrieve(url, distpath)
    progress = self.progress
    if self.verbose:
    dots = '...\n'
    else:
    dots = '...'
    if progress is not None:
    progress('Installing %s ...' % name, 'main')
    progress('Installing %s %s' % (name, dots), 'main')
    else:
    sys.stderr.write('Installing %s ...' % name)
    sys.stderr.write('Installing %s %s' % (name, dots))
    sys.stderr.flush()
    # Install in the env
    args = [context.env_exe, fn]
  5. @vsajip vsajip revised this gist Jul 12, 2013. 1 changed file with 6 additions and 5 deletions.
    11 changes: 6 additions & 5 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -15,8 +15,8 @@ class ExtendedEnvBuilder(venv.EnvBuilder):
    This builder installs setuptools and pip so that you can pip or
    easy_install other packages into the created environment.
    :param nodist: If True, setuptools is not installed into the created
    environment.
    :param nodist: If True, setuptools and pip are not installed into the
    created environment.
    :param nopip: If True, pip is not installed into the created
    environment.
    :param progress: If setuptools or pip are installed, the progress of the
    @@ -52,7 +52,8 @@ def post_setup(self, context):
    os.environ['VIRTUAL_ENV'] = context.env_dir
    if not self.nodist:
    self.install_setuptools(context)
    if not self.nopip:
    # Can't install pip without setuptools
    if not self.nopip and not self.nodist:
    self.install_pip(context)

    def reader(self, stream, context):
    @@ -151,8 +152,8 @@ def main(args=None):
    help='A directory to create the environment in.')
    parser.add_argument('--no-setuptools', default=False,
    action='store_true', dest='nodist',
    help="Don't install setuptools in the virtual "
    "environment.")
    help="Don't install setuptools or pip in the "
    "virtual environment.")
    parser.add_argument('--no-pip', default=False,
    action='store_true', dest='nopip',
    help="Don't install pip in the virtual "
  6. @vsajip vsajip revised this gist Jul 12, 2013. 1 changed file with 0 additions and 3 deletions.
    3 changes: 0 additions & 3 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,6 @@
    #
    # Copyright (C) 2013 Vinay Sajip. New BSD License.
    #
    import logging
    import os
    import os.path
    from subprocess import Popen, PIPE
    @@ -11,8 +10,6 @@
    from urllib.request import urlretrieve
    import venv

    logger = logging.getLogger(__name__)

    class ExtendedEnvBuilder(venv.EnvBuilder):
    """
    This builder installs setuptools and pip so that you can pip or
  7. @vsajip vsajip revised this gist Jul 12, 2013. 1 changed file with 20 additions and 16 deletions.
    36 changes: 20 additions & 16 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,7 @@
    #
    # Copyright (C) 2013 Vinay Sajip. New BSD License.
    #
    import logging
    import os
    import os.path
    from subprocess import Popen, PIPE
    @@ -10,16 +11,18 @@
    from urllib.request import urlretrieve
    import venv

    class DistributeEnvBuilder(venv.EnvBuilder):
    logger = logging.getLogger(__name__)

    class ExtendedEnvBuilder(venv.EnvBuilder):
    """
    This builder installs Distribute and pip so that you can pip or
    This builder installs setuptools and pip so that you can pip or
    easy_install other packages into the created environment.
    :param nodist: If True, Distribute is not installed into the created
    :param nodist: If True, setuptools is not installed into the created
    environment.
    :param nopip: If True, pip is not installed into the created
    environment.
    :param progress: If Distribute or pip are installed, the progress of the
    :param progress: If setuptools or pip are installed, the progress of the
    installation can be monitored by passing a progress
    callable. If specified, it is called with two
    arguments: a string indicating some progress, and a
    @@ -49,8 +52,9 @@ def post_setup(self, context):
    :param context: The information for the environment creation request
    being processed.
    """
    os.environ['VIRTUAL_ENV'] = context.env_dir
    if not self.nodist:
    self.install_distribute(context)
    self.install_setuptools(context)
    if not self.nopip:
    self.install_pip(context)

    @@ -82,9 +86,9 @@ def install_script(self, context, name, url):
    urlretrieve(url, distpath)
    progress = self.progress
    if progress is not None:
    progress('Installing %s' %name, 'main')
    progress('Installing %s ...' % name, 'main')
    else:
    sys.stderr.write('Installing %s ' % name)
    sys.stderr.write('Installing %s ...' % name)
    sys.stderr.flush()
    # Install in the env
    args = [context.env_exe, fn]
    @@ -103,17 +107,17 @@ def install_script(self, context, name, url):
    # Clean up - no longer needed
    os.unlink(distpath)

    def install_distribute(self, context):
    def install_setuptools(self, context):
    """
    Install Distribute in the environment.
    Install setuptools in the environment.
    :param context: The information for the environment creation request
    being processed.
    """
    url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py'
    self.install_script(context, 'distribute', url)
    # clear up the distribute archive which gets downloaded
    pred = lambda o: o.startswith('distribute-') and o.endswith('.tar.gz')
    self.install_script(context, 'setuptools', url)
    # clear up the setuptools archive which gets downloaded
    pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz')
    files = filter(pred, os.listdir(context.bin_path))
    for f in files:
    f = os.path.join(context.bin_path, f)
    @@ -148,9 +152,9 @@ def main(args=None):
    'directories.')
    parser.add_argument('dirs', metavar='ENV_DIR', nargs='+',
    help='A directory to create the environment in.')
    parser.add_argument('--no-distribute', default=False,
    parser.add_argument('--no-setuptools', default=False,
    action='store_true', dest='nodist',
    help="Don't install Distribute in the virtual "
    help="Don't install setuptools in the virtual "
    "environment.")
    parser.add_argument('--no-pip', default=False,
    action='store_true', dest='nopip',
    @@ -182,11 +186,11 @@ def main(args=None):
    parser.add_argument('--verbose', default=False, action='store_true',
    dest='verbose', help='Display the output '
    'from the scripts which '
    'install Distribute and pip.')
    'install setuptools and pip.')
    options = parser.parse_args(args)
    if options.upgrade and options.clear:
    raise ValueError('you cannot supply --upgrade and --clear together.')
    builder = DistributeEnvBuilder(system_site_packages=options.system_site,
    builder = ExtendedEnvBuilder(system_site_packages=options.system_site,
    clear=options.clear,
    symlinks=options.symlinks,
    upgrade=options.upgrade,
  8. @vsajip vsajip revised this gist Jul 10, 2013. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -110,7 +110,7 @@ def install_distribute(self, context):
    :param context: The information for the environment creation request
    being processed.
    """
    url = 'http://python-distribute.org/distribute_setup.py'
    url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py'
    self.install_script(context, 'distribute', url)
    # clear up the distribute archive which gets downloaded
    pred = lambda o: o.startswith('distribute-') and o.endswith('.tar.gz')
  9. @vsajip vsajip created this gist Jan 30, 2013.
    206 changes: 206 additions & 0 deletions pyvenvex.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,206 @@
    #
    # Copyright (C) 2013 Vinay Sajip. New BSD License.
    #
    import os
    import os.path
    from subprocess import Popen, PIPE
    import sys
    from threading import Thread
    from urllib.parse import urlparse
    from urllib.request import urlretrieve
    import venv

    class DistributeEnvBuilder(venv.EnvBuilder):
    """
    This builder installs Distribute and pip so that you can pip or
    easy_install other packages into the created environment.
    :param nodist: If True, Distribute is not installed into the created
    environment.
    :param nopip: If True, pip is not installed into the created
    environment.
    :param progress: If Distribute or pip are installed, the progress of the
    installation can be monitored by passing a progress
    callable. If specified, it is called with two
    arguments: a string indicating some progress, and a
    context indicating where the string is coming from.
    The context argument can have one of three values:
    'main', indicating that it is called from virtualize()
    itself, and 'stdout' and 'stderr', which are obtained
    by reading lines from the output streams of a subprocess
    which is used to install the app.
    If a callable is not specified, default progress
    information is output to sys.stderr.
    """

    def __init__(self, *args, **kwargs):
    self.nodist = kwargs.pop('nodist', False)
    self.nopip = kwargs.pop('nopip', False)
    self.progress = kwargs.pop('progress', None)
    self.verbose = kwargs.pop('verbose', False)
    super().__init__(*args, **kwargs)

    def post_setup(self, context):
    """
    Set up any packages which need to be pre-installed into the
    environment being created.
    :param context: The information for the environment creation request
    being processed.
    """
    if not self.nodist:
    self.install_distribute(context)
    if not self.nopip:
    self.install_pip(context)

    def reader(self, stream, context):
    """
    Read lines from a subprocess' output stream and either pass to a progress
    callable (if specified) or write progress information to sys.stderr.
    """
    progress = self.progress
    while True:
    s = stream.readline()
    if not s:
    break
    if progress is not None:
    progress(s, context)
    else:
    if not self.verbose:
    sys.stderr.write('.')
    else:
    sys.stderr.write(s.decode('utf-8'))
    sys.stderr.flush()

    def install_script(self, context, name, url):
    _, _, path, _, _, _ = urlparse(url)
    fn = os.path.split(path)[-1]
    binpath = context.bin_path
    distpath = os.path.join(binpath, fn)
    # Download script into the env's binaries folder
    urlretrieve(url, distpath)
    progress = self.progress
    if progress is not None:
    progress('Installing %s' %name, 'main')
    else:
    sys.stderr.write('Installing %s ' % name)
    sys.stderr.flush()
    # Install in the env
    args = [context.env_exe, fn]
    p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath)
    t1 = Thread(target=self.reader, args=(p.stdout, 'stdout'))
    t1.start()
    t2 = Thread(target=self.reader, args=(p.stderr, 'stderr'))
    t2.start()
    p.wait()
    t1.join()
    t2.join()
    if progress is not None:
    progress('done.', 'main')
    else:
    sys.stderr.write('done.\n')
    # Clean up - no longer needed
    os.unlink(distpath)

    def install_distribute(self, context):
    """
    Install Distribute in the environment.
    :param context: The information for the environment creation request
    being processed.
    """
    url = 'http://python-distribute.org/distribute_setup.py'
    self.install_script(context, 'distribute', url)
    # clear up the distribute archive which gets downloaded
    pred = lambda o: o.startswith('distribute-') and o.endswith('.tar.gz')
    files = filter(pred, os.listdir(context.bin_path))
    for f in files:
    f = os.path.join(context.bin_path, f)
    os.unlink(f)

    def install_pip(self, context):
    """
    Install pip in the environment.
    :param context: The information for the environment creation request
    being processed.
    """
    url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py'
    self.install_script(context, 'pip', url)

    def main(args=None):
    compatible = True
    if sys.version_info < (3, 3):
    compatible = False
    elif not hasattr(sys, 'base_prefix'):
    compatible = False
    if not compatible:
    raise ValueError('This script is only for use with '
    'Python 3.3 or later')
    else:
    import argparse

    parser = argparse.ArgumentParser(prog=__name__,
    description='Creates virtual Python '
    'environments in one or '
    'more target '
    'directories.')
    parser.add_argument('dirs', metavar='ENV_DIR', nargs='+',
    help='A directory to create the environment in.')
    parser.add_argument('--no-distribute', default=False,
    action='store_true', dest='nodist',
    help="Don't install Distribute in the virtual "
    "environment.")
    parser.add_argument('--no-pip', default=False,
    action='store_true', dest='nopip',
    help="Don't install pip in the virtual "
    "environment.")
    parser.add_argument('--system-site-packages', default=False,
    action='store_true', dest='system_site',
    help='Give the virtual environment access to the '
    'system site-packages dir.')
    if os.name == 'nt':
    use_symlinks = False
    else:
    use_symlinks = True
    parser.add_argument('--symlinks', default=use_symlinks,
    action='store_true', dest='symlinks',
    help='Try to use symlinks rather than copies, '
    'when symlinks are not the default for '
    'the platform.')
    parser.add_argument('--clear', default=False, action='store_true',
    dest='clear', help='Delete the contents of the '
    'environment directory if it '
    'already exists, before '
    'environment creation.')
    parser.add_argument('--upgrade', default=False, action='store_true',
    dest='upgrade', help='Upgrade the environment '
    'directory to use this version '
    'of Python, assuming Python '
    'has been upgraded in-place.')
    parser.add_argument('--verbose', default=False, action='store_true',
    dest='verbose', help='Display the output '
    'from the scripts which '
    'install Distribute and pip.')
    options = parser.parse_args(args)
    if options.upgrade and options.clear:
    raise ValueError('you cannot supply --upgrade and --clear together.')
    builder = DistributeEnvBuilder(system_site_packages=options.system_site,
    clear=options.clear,
    symlinks=options.symlinks,
    upgrade=options.upgrade,
    nodist=options.nodist,
    nopip=options.nopip,
    verbose=options.verbose)
    for d in options.dirs:
    builder.create(d)

    if __name__ == '__main__':
    rc = 1
    try:
    main()
    rc = 0
    except Exception as e:
    print('Error: %s' % e, file=sys.stderr)
    sys.exit(rc)