Build a packed conda env .tgz from conda environment.yml and/or pip requirements.txt

Dockerfile

# See also:_https://pythonspeed.com/articles/conda-docker-image-size/

# The conda env build stage image:
# 1. Create conda env
# 2. Optionally install conda dependencies
# 3. Optionally install pip dependencies
FROM continuumio/miniconda3 AS conda_build_env

# Create a bare conda env.
# We need at minimum python, pip, conda-pack and gcc and g++ (to build any binary pip packages later)
RUN conda create -c conda-forge -p /srv/conda_build_env \
    python=3.7 pip conda-pack gcc_linux-64 gxx_linux-64

# Create gcc and g++ symlinks in conda env, so that pip install can use them if it needs to.
# https://seanlaw.github.io/2019/01/17/pip-installing-wheels-with-conda-gcc/
RUN ln -s /srv/conda_build_env/bin/x86_64-conda-cos6-linux-gnu-gcc /srv/conda_build_env/bin/gcc && \
    ln -s /srv/conda_build_env/bin/x86_64-conda-cos6-linux-gnu-g++ /srv/conda_build_env/bin/g++

# Copy all of the current project dir into the docker build.
COPY . .

# If a conda enviroment.yml file exists, conda install any specified conda dependencies from it.
RUN (test -f environment.yml && conda env update -p /srv/conda_build_env --file environment.yml) || \
    echo "No conda environment.yml"

# If a pip requirements.txt file is specified, install it now.
RUN (test -f requirements.txt && conda run -p /srv/conda_build_env pip install -r requirements.txt) || \
    echo 'No pip requirements.txt'

RUN (test -f frozen-requirements.txt && conda run -p /srv/conda_build_env pip install -r frozen-requirements.txt) || \
    echo 'No pip frozen-requirements.txt'

# Also install the current project directory via pip.
# This will work with either setup.py or pyproject.toml (as long as [build-system] is configured properly)
RUN conda run -p /srv/conda_build_env pip install .

# Pack the conda env. This can be extracted out of the docker image later
# and used outside of docker if needed.
FROM conda_build_env as conda_packed_env

# Remove gcc and g++. shoudln't need them in the packed env.
# This makes the packed env much smaller.
RUN conda uninstall -p /srv/conda_build_env gcc_linux-64 gxx_linux-64 && \
    unlink /srv/conda_build_env/bin/gcc && \
    unlink /srv/conda_build_env/bin/g++

# Use conda-pack to create a standalone enviornment
RUN conda run -p /srv/conda_build_env conda-pack -p /srv/conda_build_env --ignore-editable-packages --format=tgz -o /srv/conda_env.tgz

# NOTE: to extract this file from the conda_packed_env build stage:
#   docker build -f ../Dockerfile --target conda_packed_env -t conda:packed_env .
#   docker create -ti --name conda_packed_env -t conda:packed_env bash
#   docker cp conda_packed_env:/srv/conda_env.tgz /tmp/conda_env.tgz
#   docker rm -f conda_packed_env

# Unpack the conda env for use by the conda_runtime image stage below.
FROM conda_packed_env as conda_env
RUN mkdir -p /srv/conda_env && \
    cd /srv/conda_env && \
    tar -xf /srv/conda_env.tgz && \
    # conda-unpack to fix up any prefixes to set them to /srv/conda_env
    /srv/conda_env/bin/conda-unpack

# The runtime-stage image; we can use Debian as the
# base image since the Conda env also includes Python
# for us.
# NOTE: This might not be needed for us at WMF data eng,
# unless we are able to run docker images.
FROM debian:buster AS conda_runtime

# Copy /srv/conda_env from the previous stage.
# In this way we keep only the final conda env
# without any of the build step leftovers.
COPY --from=conda_env /srv/conda_env /srv/conda_env




# # When image is run, run the code with the environment activated:
# SHELL ["/bin/bash", "-c"]
# ENTRYPOINT source /srv/conda_env/bin/activate && \
#            /srv/conda_env/bin/python -c "import pyarrow; print('success!')"




# to test:
# docker run -it \
# 		--rm \
# 		-v $(pwd):/srv/project \
# 		-e SKIP_DOCKER=true \
# 		-w /srv/project conda bash -c 'source /srv/conda_env/bin/activate && pip install -r ./requirements-test.txt && pytest'

Dockerfile-anaconda-wmf

# See also:_https://pythonspeed.com/articles/conda-docker-image-size/

# The conda env build stage image:
# 1. Create conda env
# 2. Optionally install conda dependencies
# 3. Optionally install pip dependencies
FROM docker-registry.wikimedia.org/wikimedia-buster AS conda_build_env

RUN apt-get update
RUN apt-get install -y anaconda-wmf

# Create a stacked conda env.
RUN /usr/lib/anaconda-wmf/bin/conda-create-stacked conda_build_env

ENV PATH="${PATH}:/usr/lib/anaconda-wmf/condabin"
ENV CONDA_ENV_PREFIX="/root/.conda/envs/conda_build_env"

# # Create gcc and g++ symlinks in conda env, so that pip install can use them if it needs to.
# # https://seanlaw.github.io/2019/01/17/pip-installing-wheels-with-conda-gcc/
# RUN ln -s /srv/conda_build_env/bin/x86_64-conda-cos6-linux-gnu-gcc /srv/conda_build_env/bin/gcc && \
#     ln -s /srv/conda_build_env/bin/x86_64-conda-cos6-linux-gnu-g++ /srv/conda_build_env/bin/g++

# Copy all of the current project dir into the docker build.
COPY . /srv/project

# If a conda enviroment.yml file exists, conda install any specified conda dependencies from it.
RUN /bin/bash -c "(cd /srv/project && test -f environment.yml && . source ${CONDA_ENV_PREFIX}/bin/activate && conda env update --file environment.yml) || echo 'No conda environment.yml'"

# If a pip requirements.txt file is specified, install it now.
RUN /bin/bash -c "(cd /srv/project && test -f requirements.txt && source ${CONDA_ENV_PREFIX}/bin/activate && pip install -r requirements.txt) ||  echo 'No pip requirements.txt'"

RUN /bin/bash -c "(cd /srv/project && test -f frozen-requirements.txt && source ${CONDA_ENV_PREFIX}/bin/activate && pip install -r frozen-requirements.txt) || echo 'No pip frozen-requirements.txt'"

# Also install the current project directory via pip.
# This will work with either setup.py or pyproject.toml (as long as [build-system] is configured properly)
RUN /bin/bash -c "cd /srv/project && source ${CONDA_ENV_PREFIX}/bin/activate && pip install ."

# Pack the conda env. This can be extracted out of the docker image later
# and used outside of docker if needed.
FROM conda_build_env as conda_packed_env

# Use conda-pack to create a standalone enviornment
RUN /bin/bash -c 'source ${CONDA_ENV_PREFIX}/bin/activate && conda-pack --ignore-editable-packages --format=tgz -o /srv/conda_env.tgz'

# NOTE: to extract this file from the conda_packed_env build stage:
#   docker build -f ../Dockerfile-anaconda-wmf --target conda_packed_env -t conda:packed_env .
#   docker create -ti --name conda_packed_env -t conda:packed_env bash
#   docker cp conda_packed_env:/srv/conda_env.tgz /tmp/conda_env.tgz
#   docker rm -f conda_packed_env