parallelpip demo
#! -*- Coding: utf-8 -*- | |
from gevent import monkey | |
monkey.patch_all() | |
import gevent | |
import time | |
from envoy import run | |
from sys import exit, argv | |
import subprocess | |
import pip | |
def syntax(): | |
print "Usage:python core.py normal|parallel [libraries]" | |
def normal_download(lib = None): | |
try: | |
if lib: | |
start = time.time() | |
print "normal download started" | |
for l in lib: | |
print "Trying to install %s"%l | |
run("pip install %s"%l) | |
return time.time() - start | |
else: | |
syntax() | |
exit() | |
except: | |
print "Unhandled exception" | |
exit() | |
def parallel_download(lib = None): | |
try: | |
if lib: | |
print "spawning using gevent" | |
jobs = [gevent.spawn(pip.call_subprocess, ["pip","install",l]) \ | |
for l in lib] | |
start = time.time() | |
print "joined all gevent, d/l started" | |
gevent.joinall(jobs) | |
for job in jobs: | |
print job.value | |
return time.time() - start | |
else: | |
syntax() | |
exit() | |
except Exception, e: | |
print "unhandled exception", e | |
exit() | |
if __name__ == "__main__": | |
if argv[1] == 'parallel': | |
print(parallel_download(argv[2:])," seconds for parallel d/l") | |
elif argv[1] == 'normal': | |
print(normal_download(argv[2:]), "seconds for normal d/l") | |
else: | |
syntax() | |
exit() |
1. Create two virtualenv named normalpip and parallelpip. | |
2. Install envoy, gevent in both the env(pip install envoy gevent) | |
3. . normalpip/bin/activate | |
4.clone the code and name as core.py. | |
5.python core.py normal flask requests | |
6. . parallelpip/bin/activate | |
7.python core.py parallel flask requests | |
sample output | |
============= | |
(normalpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py normal flask requests | |
normal download started | |
Trying to install flask | |
Trying to install requests | |
(146.34172201156616, 'seconds for normal d/l') | |
(testparallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests | |
spawning using gevent | |
joined all gevent, d/l started | |
Downloading/unpacking flask | |
Downloading Flask-0.8.tar.gz (494Kb): 494Kb downloaded | |
Running setup.py egg_info for package flask | |
warning: no files found matching '*' under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'docs' | |
warning: no previously-included files matching '*.pyo' found under directory 'docs' | |
warning: no previously-included files matching '*.pyc' found under directory 'tests' | |
warning: no previously-included files matching '*.pyo' found under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'examples' | |
warning: no previously-included files matching '*.pyo' found under directory 'examples' | |
no previously-included directories found matching 'docs/_build' | |
no previously-included directories found matching 'docs/_themes/.git' | |
Downloading/unpacking Werkzeug>=0.6.1 (from flask) | |
Downloading Werkzeug-0.8.3.tar.gz (1.1Mb): 1.1Mb downloaded | |
Running setup.py egg_info for package Werkzeug | |
warning: no files found matching '*' under directory 'werkzeug/debug/templates' | |
warning: no files found matching '*' under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'docs' | |
warning: no previously-included files matching '*.pyo' found under directory 'docs' | |
warning: no previously-included files matching '*.pyc' found under directory 'tests' | |
warning: no previously-included files matching '*.pyo' found under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'examples' | |
warning: no previously-included files matching '*.pyo' found under directory 'examples' | |
no previously-included directories found matching 'docs/_build' | |
Downloading/unpacking Jinja2>=2.4 (from flask) | |
Downloading Jinja2-2.6.tar.gz (389Kb): 389Kb downloaded | |
Running setup.py egg_info for package Jinja2 | |
warning: no previously-included files matching '*' found under directory 'docs/_build' | |
warning: no previously-included files matching '*.pyc' found under directory 'jinja2' | |
warning: no previously-included files matching '*.pyc' found under directory 'docs' | |
warning: no previously-included files matching '*.pyo' found under directory 'jinja2' | |
warning: no previously-included files matching '*.pyo' found under directory 'docs' | |
Installing collected packages: flask, Werkzeug, Jinja2 | |
Running setup.py install for flask | |
warning: no files found matching '*' under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'docs' | |
warning: no previously-included files matching '*.pyo' found under directory 'docs' | |
warning: no previously-included files matching '*.pyc' found under directory 'tests' | |
warning: no previously-included files matching '*.pyo' found under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'examples' | |
warning: no previously-included files matching '*.pyo' found under directory 'examples' | |
no previously-included directories found matching 'docs/_build' | |
no previously-included directories found matching 'docs/_themes/.git' | |
Running setup.py install for Werkzeug | |
warning: no files found matching '*' under directory 'werkzeug/debug/templates' | |
warning: no files found matching '*' under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'docs' | |
warning: no previously-included files matching '*.pyo' found under directory 'docs' | |
warning: no previously-included files matching '*.pyc' found under directory 'tests' | |
warning: no previously-included files matching '*.pyo' found under directory 'tests' | |
warning: no previously-included files matching '*.pyc' found under directory 'examples' | |
warning: no previously-included files matching '*.pyo' found under directory 'examples' | |
no previously-included directories found matching 'docs/_build' | |
Running setup.py install for Jinja2 | |
warning: no previously-included files matching '*' found under directory 'docs/_build' | |
warning: no previously-included files matching '*.pyc' found under directory 'jinja2' | |
warning: no previously-included files matching '*.pyc' found under directory 'docs' | |
warning: no previously-included files matching '*.pyo' found under directory 'jinja2' | |
warning: no previously-included files matching '*.pyo' found under directory 'docs' | |
Successfully installed flask Werkzeug Jinja2 | |
Cleaning up... | |
Downloading/unpacking requests | |
Downloading requests-0.10.6.tar.gz (61Kb): 61Kb downloaded | |
Running setup.py egg_info for package requests | |
warning: no files found matching 'test_requests.py' | |
Downloading/unpacking certifi>=0.0.7 (from requests) | |
Downloading certifi-0.0.8.tar.gz (118Kb): 118Kb downloaded | |
Running setup.py egg_info for package certifi | |
Downloading/unpacking chardet>=1.0.0 (from requests) | |
Downloading chardet-1.0.1.tar.gz (156Kb): 156Kb downloaded | |
Running setup.py egg_info for package chardet | |
Installing collected packages: requests, certifi, chardet | |
Running setup.py install for requests | |
warning: no files found matching 'test_requests.py' | |
Running setup.py install for certifi | |
Running setup.py install for chardet | |
Successfully installed requests certifi chardet | |
Cleaning up... | |
None | |
None | |
(83.12853598594666, ' seconds for parallel d/l') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
The parallel version gives me errors when I run with many dependencies that have a common set of transitive dependencies. The error pops up when multiple projects try to simultaneously installed a transitive dependency. Any ideas how to alleviate/fix that problem?