README.md

Files to profile memory usage of conda in an environment similar to conda/conda#7787

Instructions

Build the docker image using:

docker build -t mem_test .

This installs a development version of conda and creates an environment similar to the one in the GitHub issue.

Run the docker container

docker run -v `pwd`:/io -it mem_test

Inside the container run the bash script to collect the memory profiles and copy them to the host system.

/io/run_mem_profiles.sh
cp ~/conda/*.mprof /io/

Dockerfile

FROM centos:6

RUN yum install -y \
        bzip2 \
        curl \
        git \
    yum clean all

RUN cd ~ && \
    git clone https://github.com/conda/conda && \
    cd conda && \
    git checkout 4.6.0b0 && \
    . dev/start ./devenv 3.5 && \
    git checkout 4.5.11 && \
    conda install -qy python="${pyver}" memory_profiler

COPY pkg_list.txt /root/pkg_list.txt

RUN . ~/conda/devenv/etc/profile.d/conda.sh && conda activate base && \
    conda create -nx --file ~/pkg_list.txt

COPY requirements.txt /root/requirements.txt
RUN . ~/conda/devenv/etc/profile.d/conda.sh && conda activate base && \
    conda activate x && \
    pip install -r /root/requirements.txt


RUN . ~/conda/devenv/etc/profile.d/conda.sh && conda activate base && \
    conda list -n x --show-channel-url

pkg_list.txt

conda-canary::conda=4.6.0b0=py35_0
conda-canary::requests=2.19.1=py35_0
conda-forge::affine=2.1.0=py_1
conda-forge::alabaster=0.7.11=py_3
conda-forge::altair=2.2.2=py_0
conda-forge::anaconda-client=1.6.3=py35_0
conda-forge::appdirs=1.4.3=py_1
conda-forge::args=0.1.0=py35_0
conda-forge::arrow-cpp=0.9.0=py35_7
conda-forge::asn1crypto=0.22.0=py35_0
conda-forge::astropy=2.0.3=py35_0
conda-forge::attrs=17.3.0=py_0
conda-forge::automat=0.7.0=py_1
conda-forge::babel=2.6.0=py_1
conda-forge::backports.functools_lru_cache=1.5=py35_0
conda-forge::backports.weakref=1.0rc1=py35_0
conda-forge::backports=1.0=py35_1
conda-forge::beautifulsoup4=4.6.0=py35_0
conda-forge::bitarray=0.8.1=py35_0
conda-forge::bkcharts=0.2=py35_0
conda-forge::blas=1.1=openblas
conda-forge::bleach=1.5.0=py35_0
conda-forge/label/broken::blinker=1.4=py35_0
conda-forge::bokeh=0.12.9=py35_0
conda-forge::boost-cpp=1.66.0=1
conda-forge::boost=1.66.0=py35_1
conda-forge::boto3=1.4.7=py35_0
conda-forge::botocore=1.5.92=py35_0
conda-forge::bottleneck=1.2.1=np113py35_0
conda-forge::branca=0.3.0=py_0
conda-forge::bzip2=1.0.6=1
conda-forge::ca-certificates=2018.8.24=ha4d7672_0
conda-forge::cachetools=2.0.1=py_0
conda-forge::cairo=1.14.10=0
conda-forge::cartopy=0.16.0=py35_0
conda-forge::certifi=2018.8.24=py35_1
conda-forge::chardet=3.0.4=py35_0
conda-forge::click-plugins=1.0.3=py35_0
conda-forge::clickhouse-driver=0.0.10=py35_0
conda-forge::cligj=0.4.0=py35_0
conda-forge::clint=0.5.1=py35_0
conda-forge::cloudpickle=0.4.0=py35_0
conda-forge::clyent=1.2.2=py35_0
conda-forge::colorama=0.3.9=py35_0
conda-forge::conda-build-all=1.1.1=py35_1
conda-forge::conda-build=3.10.9=py35_0
conda-forge::conda-forge-pinning=2018.07.01=0
conda-forge::conda-smithy=3.1.6=py_0
conda-forge::conda-verify=2.0.0=py35_0
conda-forge::constantly=15.1.0=py_0
conda-forge::coverage=4.4.1=py35_0
conda-forge::cryptography=1.9=py35_0
conda-forge::curl=7.54.1=0
conda-forge::cycler=0.10.0=py35_0
conda-forge::cytoolz=0.9.0.1=py35_0
conda-forge::dask-core=0.17.4=py_0
conda-forge::dask=0.17.4=py_0
conda-forge::dbus=1.10.22=0
conda-forge::decorator=4.1.2=py35_0
conda-forge::descartes=1.1.0=py35_0
conda-forge::distributed=1.21.8=py35_0
conda-forge::django=2.0.1=py35_0
conda-forge::docopt=0.6.2=py35_0
conda-forge::doctr=1.7.3=py35_0
conda-forge::docutils=0.14=py35_0
conda-forge::expat=2.2.5=0
conda-forge::fastavro=0.17.10=py35_1
conda-forge::fastparquet=0.1.3=py35_0
conda-forge::feedgenerator=1.9=py35_0
conda-forge::ffmpeg=3.2.4=1
conda-forge::filelock=2.0.6=py35_0
conda-forge::fiona=1.7.11=py35_3
conda-forge::fontconfig=2.12.6=0
conda-forge::freetype=2.8.1=0
conda-forge::freexl=1.0.2=2
conda-forge::gdal=2.2.4=py35_0
conda-forge::gensim=3.0.1=py35_0
conda-forge::geoalchemy2=0.4.0=py35_1
conda-forge::geopandas=0.4.0=py_1
conda-forge::geoplot=0.1.2=1
conda-forge::geopy=1.11.0=py35_0
conda-forge::geos=3.6.2=1
conda-forge::geotiff=1.4.2=1
conda-forge::gettext=0.19.7=1
conda-forge::ghp-import=0.5.5=py35_0
conda-forge::giflib=5.1.4=0
conda-forge::gitdb2=2.0.2=py35_0
conda-forge::gitpython=2.1.5=py35_0
conda-forge::glib=2.55.0=0
conda-forge::glob2=0.6=py_0
conda-forge::gmp=6.1.2=0
conda-forge::google-api-core=1.1.1=py_0
conda-forge::google-auth=1.4.1=py_0
conda-forge::google-cloud-bigquery=1.1.0=py_0
conda-forge::google-cloud-core=0.28.1=py_0
conda-forge::google-resumable-media=0.3.1=py_0
conda-forge::googleapis-common-protos=1.5.3=py35_0
conda-forge::graphite2=1.3.11=0
conda-forge::graphviz=2.38.0=7
conda-forge::gst-plugins-base=1.8.0=0
conda-forge::gstreamer=1.8.0=1
conda-forge::h5netcdf=0.4.0=py35_0
conda-forge::h5py=2.7.1=py35_3
conda-forge::harfbuzz=1.7.6=0
conda-forge::hdbscan=0.8.12=py35_1
conda-forge::hdf4=4.2.12=0
conda-forge::hdf5=1.10.1=2
conda-forge::heapdict=1.0.0=py35_0
conda-forge::html5lib=0.9999999=py35_0
conda-forge::hyperlink=17.3.1=py_0
conda-forge::icu=58.1=1
conda-forge::imageio=2.2.0=py35_0
conda-forge::imagesize=1.0.0=py_1
conda-forge::imbalanced-learn=0.3.2=py_0
conda-forge::impyla=0.14.0=py35_0
conda-forge::incremental=17.5.0=py_0
conda-forge::ipykernel=4.6.1=py35_0
conda-forge::ipympl=0.1.1=py35_0
conda-forge::ipython=6.1.0=py35_0
conda-forge::ipython_genutils=0.2.0=py35_0
conda-forge::ipywidgets=7.0.0=py35_0
conda-forge::jasper=1.900.1=4
conda-forge::jeepney=0.3.1=py_0
conda-forge::jinja2=2.9.6=py35_0
conda-forge::jmespath=0.9.3=py35_0
conda-forge::joblib=0.11=py35_0
conda-forge::jpeg=9b=0
conda-forge::json-c=0.12.1=0
conda-forge::jsonschema=2.6.0=py35_0
conda-forge::jupyter=1.0.0=py35_0
conda-forge::jupyter_client=5.2.2=py35_0
conda-forge::jupyter_console=5.1.0=py35_0
conda-forge::jupyter_contrib_core=0.3.3=py35_0
conda-forge::jupyter_core=4.4.0=py_0
conda-forge::jupyterlab=0.33.12=py35_0
conda-forge::jupyterlab_launcher=0.11.2=py_0
conda-forge::kealib=1.4.7=4
conda-forge::keras=2.0.6=py35_0
conda-forge::keyring=13.2.1=py35_0
conda-forge::kiwisolver=1.0.1=py35_1
conda-forge::krb5=1.14.2=0
conda-forge::libdap4=3.18.3=2
conda-forge::libffi=3.2.1=3
conda-forge/label/broken::libgdal=2.2.4=0
conda-forge::libgpuarray=0.7.0=0
conda-forge/label/broken::libiconv=1.15=0
conda-forge/label/broken::libkml=1.3.0=6
conda-forge::libnetcdf=4.4.1.1=10
conda-forge::libpng=1.6.34=0
conda-forge::libpq=9.6.3=0
conda-forge::libprotobuf=3.5.2=0
conda-forge::libsodium=1.0.10=0
conda-forge::libspatialindex=1.8.5=1
conda-forge::libspatialite=4.3.0a=19
conda-forge::libssh2=1.8.0=1
conda-forge::libstdcxx-ng=7.2.0=hdf63c60_3
conda-forge::libtiff=4.0.9=0
conda-forge::libtool=2.4.6=0
conda-forge::libwebp=0.5.2=7
conda-forge::libxcb=1.12=1
conda-forge::libxml2=2.9.4=4
conda-forge::libxslt=1.1.29=5
conda-forge::lifelines=0.9.4=py35_0
conda-forge::locket=0.2.0=py35_1
conda-forge::lxml=3.8.0=py35_0
conda-forge::mahotas=1.4.3=np113py35_1
conda-forge::markupsafe=1.0=py35_0
conda-forge::matplotlib=2.2.2=py35_0
conda-forge::mercantile=0.10.0=py35_0
conda-forge::metpy=0.7.0=py35_0
conda-forge::mistune=0.7.4=py35_0
conda-forge::mizani=0.4.6=py_0
conda-forge::mock=2.0.0=py35_0
conda-forge::mpi4py=3.0.0=py35_openmpi_3
conda-forge::mpi=1.0=openmpi
conda-forge::mplleaflet=0.0.5=py35_1
conda-forge::msgpack-python=0.4.8=py35_0
conda-forge::multipledispatch=0.5.0=py35_0
conda-forge::munch=2.2.0=py35_0
conda-forge::nbconvert=5.2.1=py35_1
conda-forge::nbformat=4.3.0=py35_0
conda-forge::ncurses=5.9=10
conda-forge::nettle=3.4=0
conda-forge::networkx=1.11=py35_0
conda-forge::nibabel=2.2.1=py35_0
conda-forge::nilearn=0.4.1=py_0
conda-forge::nltk=3.2.5=py_0
conda-forge::nose=1.3.7=py35_2
conda-forge::notebook=5.4.1=py35_0
conda-forge::numexpr=2.6.2=np113py35_0
conda-forge::numpydoc=0.8.0=py_1
conda-forge::oauthlib=2.0.6=py_0
conda-forge::olefile=0.44=py35_0
conda-forge::openblas=0.2.20=7
conda-forge::openjpeg=2.3.0=2
conda-forge::openmpi=3.1.0=h26a2512_3
conda-forge::openssl=1.0.2p=h470a237_0
conda-forge::packaging=16.8=py35_0
conda-forge::palettable=3.1.1=py_0
conda-forge::pandoc=1.19.2=0
conda-forge::pandocfilters=1.4.1=py35_0
conda-forge::pango=1.40.14=0
conda-forge::parquet-cpp=1.4.0=0
conda-forge::partd=0.3.8=py35_0
conda-forge::patchelf=0.9=2
conda-forge::patsy=0.4.1=py35_0
conda-forge::pbr=3.1.1=py35_0
conda-forge::pcre=8.39=0
conda-forge::pelican=3.7.1=py35_0
conda-forge::pexpect=4.2.1=py35_0
conda-forge::pickleshare=0.7.4=py35_0
conda-forge::pint=0.8.1=py35_0
conda-forge/label/broken::pip=9.0.1=py35_0
conda-forge::pixman=0.34.0=0
conda-forge::pkginfo=1.4.1=py35_0
conda-forge::plotnine=0.4.0=py_0
conda-forge::ply=3.11=py35_0
conda-forge::poppler-data=0.4.7=0
conda-forge::poppler=0.61.1=3
conda-forge::proj4=4.9.3=4
conda-forge::prometheus_client=0.3.0=py_0
conda-forge::prompt_toolkit=1.0.15=py35_0
conda-forge::protobuf=3.5.2=py35_0
conda-forge::psycopg2=2.7.3.1=py35_0
conda-forge::ptyprocess=0.5.2=py35_0
conda-forge::py=1.4.34=py35_0
conda-forge::pyarrow=0.9.0=py35_1
conda-forge::pyasn1-modules=0.2.1=py_0
conda-forge::pycosat=0.6.3=py35_0
conda-forge::pycrypto=2.6.1=py35_1
conda-forge::pydicom=1.0.2=py_0
conda-forge/label/broken::pyepsg=0.3.2=py35_0
conda-forge::pygithub=1.29=py35_0
conda-forge::pygments=2.2.0=py35_0
conda-forge::pyjwt=1.5.3=py_0
conda-forge::pymysql=0.8.0=py_0
conda-forge::pyopenssl=17.2.0=py35_0
conda-forge::pyparsing=2.2.0=py35_0
conda-forge/label/broken::pyproj=1.9.5.1=py35_0
conda-forge::pyqt=5.6.0=py35_4
conda-forge::pysal=1.14.3=py35_0
conda-forge::pyshp=1.2.12=py_0
conda-forge::pysocks=1.6.7=py35_0
conda-forge::pytables=3.4.2=py35_7
conda-forge::pytest-cov=2.5.1=py35_0
conda-forge::pytest=3.2.1=py35_0
conda-forge::python-crfsuite=0.9.2=py35_0
conda-forge::python-dateutil=2.6.1=py35_0
conda-forge::python-hdfs=2.1.0=py35_0
conda-forge::python=3.5.5=0
conda-forge::pytz=2017.2=py35_0
conda-forge::pywavelets=0.5.2=np113py35_0
conda-forge::pyyaml=3.12=py35_1
conda-forge::pyzmq=16.0.2=py35_3
conda-forge::qt=5.6.2=7
conda-forge::qtawesome=0.4.4=pyh8a2030e_1
conda-forge::qtconsole=4.3.1=py35_0
conda-forge::qtpy=1.4.2=pyh8a2030e_1
conda-forge::rasterio=0.36.0=py35_3
conda-forge::readline=7.0=0
conda-forge::requests-oauthlib=0.8.0=py35_1
conda-forge::requests-toolbelt=0.8.0=py35_0
conda-forge::rope=0.10.7=py_1
conda-forge::rsa=3.4.2=py35_0
conda-forge::rtree=0.8.3=py35_0
conda-forge::ruamel.yaml=0.15.31=py35_0
conda-forge::ruamel_yaml=0.11.14=py35_0
conda-forge::s3transfer=0.1.11=py35_0
conda-forge::scikit-image=0.13.0=py35_1
conda-forge::seaborn=0.8.0=py35_0
conda-forge::secretstorage=3.0.1=py35_0
conda-forge::send2trash=1.4.2=py_0
conda-forge::service_identity=17.0.0=py_0
conda-forge::setuptools=36.2.2=py35_0
conda-forge::shapely=1.6.4=py35_0
conda-forge::simplegeneric=0.8.1=py35_0
conda-forge::sip=4.18=py35_1
conda-forge::six=1.10.0=py35_1
conda-forge::smart_open=1.5.3=py35_0
conda-forge::smmap2=2.0.3=py35_0
conda-forge::snowballstemmer=1.2.1=py_1
conda-forge::snuggs=1.4.1=py35_0
conda-forge::sortedcontainers=1.5.7=py35_0
conda-forge::sphinx-gallery=0.2.0=py35_0
conda-forge::sphinx=1.7.8=py35_0
conda-forge::sphinxcontrib-websupport=1.0.1=py35_0
conda-forge::spyder-kernels=0.2.4=py_2
conda-forge::spyder=3.3.1=py35_1
conda-forge::sqlalchemy=1.1.13=py35_0
conda-forge::sqlite=3.20.1=2
conda-forge::statsmodels=0.8.0
conda-forge::tblib=1.3.2=py35_0
conda-forge::tensorboard=0.4.0rc3=py35_0
conda-forge::tensorflow=1.4.0=py35_0
conda-forge::terminado=0.8.1=py35_0
conda-forge::testpath=0.3.1=py35_0
conda-forge/label/broken::theano=0.9.0=py35_0
conda-forge::thrift=0.10.0=py35_0
conda-forge::thriftpy=0.3.9=py35_0
conda-forge::tk=8.6.7=0
conda-forge::toolz=0.8.2=py35_0
conda-forge::tornado=4.5.1=py35_0
conda-forge::traitlets=4.3.2=py35_0
conda-forge::turbodbc=2.6.0=py35_0
conda-forge::twine=1.8.1=py35_1
conda-forge::twython=3.6.0=py35_0
conda-forge::typed-ast=1.1.0=py35_0
conda-forge::typing=3.6.6=py35_0
conda-forge::unidecode=0.04.21=py35_0
conda-forge::unixodbc=2.3.4=1
conda-forge::urllib3=1.22=py35_0
conda-forge/label/broken::vega=2.0.0=py35_0
conda-forge::vega_datasets=0.5.0=py_0
conda-forge::vincent=0.4.4=py35_0
conda-forge::wcwidth=0.1.7=py35_0
conda-forge::webencodings=0.5=py35_0
conda-forge::werkzeug=0.12.2=py35_0
conda-forge::wheel=0.29.0=py35_0
conda-forge::widgetsnbextension=3.0.2=py35_0
conda-forge::x264=20131217=3
conda-forge::xerces-c=3.2.0=0
conda-forge::xlrd=1.0.0=py35_1
conda-forge::xlwt=1.2.0=py35_0
conda-forge::xorg-libxau=1.0.8=3
conda-forge::xorg-libxdmcp=1.1.2=3
conda-forge::xz=5.2.3=0
conda-forge::yaml=0.1.6=0
conda-forge::zeromq=4.2.1=1
conda-forge::zict=0.1.3=py_0
conda-forge::zlib=1.2.11=0
conda-forge::zope.interface=4.5.0=py35h470a237_0
damianavila82::rise=5.2.0=py35_0
defaults::boto=2.48.0=py35_0
defaults::bz2file=0.98=py35_0
defaults::conda-env=2.6.0=h36134e3_1
defaults::futures=3.0.3=py35_0
defaults::jedi=0.11.1=py35_0
defaults::libgcc-ng=7.2.0=h7cc24e2_2
defaults::libgfortran=3.0.0=1
defaults::llvmlite=0.19.0=py35_0
defaults::lzo=2.06=0
defaults::markdown=2.6.9=py35_0
defaults::numba=0.34.0=np113py35_0
defaults::parso=0.1.1=py35h1b200a3_0
defaults::pomegranate=0.8.1=py35h5d3529e_0
defaults::pykerberos=1.2.1=py35h14c3975_0
defaults::requests-kerberos=0.12.0=py35_0
defaults::twisted=18.7.0=py35h14c3975_1
defaults::util-linux=2.21=0

requirements.txt

astroid==1.5.3
cchardet==1.1.3
cffi==1.7.0
click==6.7
colored==1.3.5
datapackage==0.8.8
enboard==0.1
entrypoints==0.2.2
esda==1.0.1.dev0
et-xmlfile==1.0.1
ezodf==0.3.2
fakestockdata==0.0.1
flake8==3.5.0
flit==0.12.3
ftfy==5.0.2
future==0.16.0
goodtables==1.0.0a8
idna==2.1
ijson==2.3
ircelsos==0.2.2
isodate==0.5.4
isort==4.2.15
jdcal==1.3
jsonlines==1.1.0
jsontableschema==0.10.0
jupyter-contrib-nbextensions
kml2geojson==4.0.0
langcodes==1.3.0
lazy-object-proxy==1.3.1
libpysal==3.0.4
linear-tsv==1.0.0
mapclassify==1.0.1
mccabe==0.6.1
memory-profiler==0.51.0
mgwr==1.0.0
nbconvert-reportlab==0.1
nbdime==0.1.0
notedown==1.5.0
numbannotate==0.0.3
objgraph==3.4.0
openpyxl==2.4.5
owslib==0.9.0
pandas==0.19.1
pandoc-attributes==0.1.7
pdfrw==0.2
perf==1.4
pillow==3.4.1
proselint==0.8.0
pudb==2017.1.2
pyasn1==0.1.9
pycda==0.1.15
pycodestyle==2.3.1
pycparser==2.14
pyflakes==1.6.0
pykml==0.1.0
pylint==1.7.4
pyopenms==2.3.0.3
pytoml==0.1.14
rasterstats==0.12.0
requests-download
rfc3986==0.4.1
rst2ipynb==0.2.3
simplejson==3.13.2
smopy==0.0.6
snakeviz==0.4.1
spglm==1.0.2
spreg==1.0.3
tabulator==0.14.2
unicodecsv==0.14.1
urwid==1.3.1
versioneer==0.18
vprof==0.33
wrapt==1.10.11
yapf==0.20.0
zipfile36==0.1.3

run_mem_profiles.sh

. ~/conda/devenv/etc/profile.d/conda.sh
conda activate base

# preload repodata cache
conda config --set local_repodata_ttl 999999
conda clean -yi
conda search -c conda-canary -c conda-forge -c defaults conda >/dev/null

# run memory profiles for
# conda update -nx pyarrow
# conda install -nx conda=4.5
tags=(4.5.11 4.6.0b0 4.6.0b1)
cd ~/conda
for tag in "${tags[@]}" ; do
    git checkout "${tag}"
    rm -f ./devenv/pkgs/cache/*.q
    sed -Ei.bak \
        's/( *)(def get_reduced_index\(self, specs\):.*)/\1\2\n\1    specs = sorted(specs, key=str)/' \
    ./conda/resolve.py

    mprof run --include-children -o "${tag}_pyarrow.mprof" \
        conda update -nx --dry-run -vv -c conda-canary -c conda-forge -c defaults pyarrow \
    2>&1 | grep '^DEBUG'

    mprof run --include-children -o "${tag}_conda.mprof" \
        conda install -nx --dry-run -vv -c conda-canary -c conda-forge -c defaults conda=4.5 \
    2>&1 | grep '^DEBUG'

    git checkout ./conda/resolve.py
done