ryanjulian/Dockerfile

## Dockerfile
ARG PARENT_IMAGE=ubuntu:18.04

# Garage base target ###########################################################
FROM $PARENT_IMAGE AS garage-18.04

# http://bugs.python.org/issue19846
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
ENV LANG C.UTF-8

# apt dependencies
RUN \
  apt-get -y -q update && \
  # Prevents debconf from prompting for user input
  # See https://github.com/phusion/baseimage-docker/issues/58
  DEBIAN_FRONTEND=noninteractive apt-get install -y \
    # Dockerfile deps
    wget \
    unzip \
    git \
    curl \
    # For building glfw
    cmake \
    xorg-dev \
    # mujoco_py
    # See https://github.com/openai/mujoco-py/blob/master/Dockerfile
    libglew-dev \
    libosmesa6-dev \
    patchelf \
    # OpenAI baselines
    libopenmpi-dev \
    # virtualenv
    libbz2-dev \
    libreadline-dev \
    libssl-dev \
    libsqlite3-dev \
    # ruby
    ruby-full && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/*

# Build GLFW because the Ubuntu 18.04 version is too old
# See https://github.com/glfw/glfw/issues/1004
RUN wget https://github.com/glfw/glfw/releases/download/3.3/glfw-3.3.zip && \
  unzip glfw-3.3.zip && \
  rm glfw-3.3.zip && \
  cd glfw-3.3 && \
  mkdir glfw-build && \
  cd glfw-build && \
  cmake -DBUILD_SHARED_LIBS=ON -DGLFW_BUILD_EXAMPLES=OFF -DGLFW_BUILD_TESTS=OFF -DGLFW_BUILD_DOCS=OFF .. && \
  make -j"$(nproc)" && \
  make install && \
  cd ../../ && \
  rm -rf glfw-3.3

ARG user=garage-user
ARG uid=999
RUN groupadd -g $uid $user && \
    useradd -m -r -u $uid -g $user $user && \
    chown -R $user:$user /home/$user
ENV USER $user
USER $user
ENV HOME /home/$user
ENV PYENV_ROOT "$HOME/.pyenv"
ENV PATH "$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH:$HOME/.local/bin"
ENV PATH_NO_VENV $PATH
WORKDIR $HOME

RUN curl https://pyenv.run | bash && \
  pyenv install 3.6.10 && \
  pyenv global 3.6.10 && \
  pip install virtualenv && \
  rm -r $HOME/.cache/pip

# MuJoCo 2.0 (for dm_control)
RUN mkdir -p $HOME/.mujoco && \
  wget https://www.roboti.us/download/mujoco200_linux.zip -O mujoco.zip --no-check-certificate && \
  unzip mujoco.zip -d $HOME/.mujoco && \
  rm mujoco.zip && \
  ln -s $HOME/.mujoco/mujoco200_linux $HOME/.mujoco/mujoco200
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$HOME/.mujoco/mujoco200/bin

# Fixes Segmentation Fault
# See: https://github.com/openai/mujoco-py/pull/145#issuecomment-356938564
ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libGLEW.so

# Set MuJoCo rendering mode (for dm_control)
ENV MUJOCO_GL "glfw"


# Create virtualenv
ENV VIRTUAL_ENV $HOME/venv
RUN python -m virtualenv $VIRTUAL_ENV
ENV PATH "$VIRTUAL_ENV/bin:$PATH"

# Prevent pip from complaining about available upgrades inside virtualenv
RUN pip install --upgrade pip setuptools wheel && \
  rm -r $HOME/.cache/pip

# We need a MuJoCo key to install mujoco_py
# In this step only the presence of the file mjkey.txt is required, so we only
# create an empty file
RUN touch $HOME/.mujoco/mjkey.txt

COPY --chown=$user:$user docker/entrypoint-runtime.sh $HOME/code/garage/docker/entrypoint-runtime.sh
ENTRYPOINT ["code/garage/docker/entrypoint-runtime.sh"]

# Headless machine with xvfb, i.e. so rendering works ##########################
FROM garage-18.04 AS garage-headless-18.04
USER root
RUN \
  apt-get -y -q update && \
  # Prevents debconf from prompting for user input
  # See https://github.com/phusion/baseimage-docker/issues/58
  DEBIAN_FRONTEND=noninteractive apt-get install -y \
    # Dummy X server
    xvfb \
    pulseaudio && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/* && \
  mkdir -p -m 1777 /tmp/.X11-unix

ARG user=garage-user
ARG uid=999
USER $user

# Ready, set, go.
COPY --chown=$user:$user docker/entrypoint-headless.sh $HOME/code/garage/docker/entrypoint-headless.sh
ENTRYPOINT ["code/garage/docker/entrypoint-headless.sh"]


# Special setup for garage developers, which installs garage from source #######
# Assumes $PWD is the garage repository.
FROM garage-headless-18.04 AS garage-dev-18.04

# apt dependencies
USER root
RUN \
  apt-get -y -q update && \
  # Prevents debconf from prompting for user input
  # See https://github.com/phusion/baseimage-docker/issues/58
  DEBIAN_FRONTEND=noninteractive apt-get install -y \
    # graphviz for autoapi documentation
    graphviz && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/*

ARG user=garage-user
ARG uid=999
USER $user

# Copy over just setup.py first, so the Docker cache doesn't expire until
# dependencies change
#
# Files needed to run setup.py
# - README.md
# - VERSION
# - scripts/garage
# - src/garage/__init__.py
# - setup.py
COPY --chown=$user:$user README.md $HOME/code/garage/README.md
COPY --chown=$user:$user VERSION $HOME/code/garage/VERSION
COPY --chown=$user:$user scripts/garage $HOME/code/garage/scripts/garage
COPY --chown=$user:$user src/garage/__init__.py $HOME/code/garage/src/garage/__init__.py
COPY --chown=$user:$user setup.py $HOME/code/garage/setup.py
WORKDIR $HOME/code/garage

# Pre-build pre-commit env
COPY --chown=$user:$user .pre-commit-config.yaml $HOME/code/garage
RUN git init && \
  pip install pre-commit && \
  pre-commit install && \
  pre-commit install-hooks && \
  rm -r $HOME/.cache/pip

# Install deps (but not the code)
RUN pip install --upgrade pip setuptools wheel && \
  pip install .[all,dev] && \
  rm $HOME/.mujoco/mjkey.txt && \
  rm -r $HOME/.cache/pip

# Add code stub last (ensures code changes have the shortest builds)
COPY --chown=$user:$user . $HOME/code/garage

# Build and install the sdist
RUN python setup.py sdist && \
    cp dist/*.tar.gz dist/garage.tar.gz && \
    pip install dist/garage.tar.gz[all,dev]

RUN cd benchmarks && python setup.py sdist && \
    cp dist/*.tar.gz dist/benchmarks.tar.gz && \
    pip install dist/benchmarks.tar.gz

ENTRYPOINT ["docker/entrypoint-headless.sh"]

# Nvidia target ################################################################
FROM garage-dev-18.04 AS garage-nvidia-18.04
ENTRYPOINT ["docker/entrypoint-runtime.sh"]

# Test target ##################################################################
FROM garage-dev-18.04 AS garage-test-18.04
CMD nice -n 11 pytest -v -n auto -m 'not huge and not flaky' --durations=20

## Makefile
SHELL := /bin/bash

.PHONY: help test check docs ci-job-normal ci-job-large ci-job-nightly \
	ci-job-verify-envs ci-verify-envs-conda ci-verify-envs-pipenv build-ci \
	build-headless build-nvidia run-ci run-headless run-nvidia assert-docker

.DEFAULT_GOAL := help

# Path in host where the experiment data obtained in the container is stored
DATA_PATH ?= $(shell pwd)/data
# Set the environment variable MJKEY with the contents of the file specified by
# MJKEY_PATH.
MJKEY_PATH ?= ${HOME}/.mujoco/mjkey.txt


build-test: TAG ?= rlworkgroup/garage-test
build-test: assert-docker-version docker/Dockerfile
	docker build \
		-f docker/Dockerfile \
		--cache-from rlworkgroup/garage-test:latest \
		--cache-from rlworkgroup/garage-headless:latest \
		--target garage-test-18.04 \
		-t ${TAG} \
		${BUILD_ARGS} \
		.

test:  ## Run the garage-test-18.04 docker target
test: RUN_ARGS = --memory 7500m --memory-swap 7500m
test: TAG ?= rlworkgroup/garage-test
test: CONTAINER_NAME ?= ''
test: build-test
	@echo "Running test suite..."
	docker run \
		-it \
		--rm \
		-e MJKEY="$$(cat $(MJKEY_PATH))" \
		--name ${CONTAINER_NAME} \
		${RUN_ARGS} \
		${TAG}

docs:  ## Build HTML documentation
docs:
	@pushd docs && make html && popd
	@python -c 'import os, webbrowser; webbrowser.open("file://" + os.path.realpath("docs/_build/html/index.html"))'

ci-job-precommit: assert-docker
	scripts/travisci/check_precommit.sh
	@pushd docs && make doctest clean && popd

ci-job-normal: assert-docker
	[ ! -f $(MJKEY_PATH) ] || mv $(MJKEY_PATH) $(MJKEY_PATH).bak
	pytest --cov=garage --cov-report=xml -m \
	    'not nightly and not huge and not flaky and not large and not mujoco and not mujoco_long' --durations=20
	for i in {1..5}; do \
		bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
			|| echo 'Retrying...' && sleep 30 && continue; \
		exit 1; \
	done

ci-job-large: assert-docker
	[ ! -f $(MJKEY_PATH) ] || mv $(MJKEY_PATH) $(MJKEY_PATH).bak
	pytest --cov=garage --cov-report=xml -m 'large and not flaky' --durations=20
	for i in {1..5}; do \
		bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
			|| echo 'Retrying...' && sleep 30 && continue; \
		exit 1; \
	done

ci-job-mujoco: assert-docker
	pytest --cov=garage --cov-report=xml -m 'mujoco and not flaky' --durations=20
	for i in {1..5}; do \
		bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
			|| echo 'Retrying...' && sleep 30 && continue; \
		exit 1; \
	done

ci-job-mujoco-long: assert-docker
	pytest --cov=garage --cov-report=xml -m 'mujoco_long and not flaky' --durations=20
	for i in {1..5}; do \
		bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
			|| echo 'Retrying...' && sleep 30 && continue; \
		exit 1; \
	done

ci-job-nightly: assert-docker
	pytest -m nightly

ci-job-verify-envs: assert-docker ci-job-verify-envs-pipenv ci-job-verify-envs-conda

ci-job-verify-envs-conda: assert-docker
ci-job-verify-envs-conda: CONDA_ROOT := $$HOME/miniconda
ci-job-verify-envs-conda: CONDA := $(CONDA_ROOT)/bin/conda
ci-job-verify-envs-conda: GARAGE_BIN = $(CONDA_ROOT)/envs/garage-ci/bin
ci-job-verify-envs-conda:
	touch $(MJKEY_PATH)
	wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
	bash miniconda.sh -b -p $(CONDA_ROOT)
	hash -r
	$(CONDA) config --set always_yes yes --set changeps1 no
	# Related issue: https://github.com/conda/conda/issues/9105
	# Fix in conda: https://github.com/conda/conda/pull/9014
	# https://repo.continuum.io/miniconda/ doesn't have the script for 4.7.12 yet,
	# so CI fetches 4.7.10 and runs into the above issue when trying to update conda
	$(CONDA) install -c anaconda setuptools
	$(CONDA) update -q conda
	$(CONDA) init
	# Useful for debugging any issues with conda
	$(CONDA) info -a
	$(CONDA) create -n garage-ci python=3.6 pip -y
	$(GARAGE_BIN)/pip install --upgrade pip setuptools
	$(GARAGE_BIN)/pip install dist/garage.tar.gz[all,dev]
	# pylint will verify all imports work
	$(GARAGE_BIN)/pylint --disable=all --enable=import-error garage

# The following two lines remove the Dockerfile's built-in virtualenv from the
# path, so we can test with pipenv directly
ci-job-verify-envs-pipenv: assert-docker
ci-job-verify-envs-pipenv: export PATH=$(shell echo $$PATH_NO_VENV)
ci-job-verify-envs-pipenv: export VIRTUAL_ENV=
ci-job-verify-envs-pipenv: export PIPENV_MAX_RETRIES=2  # number of retries for network requests. Default is 0
ci-job-verify-envs-pipenv:
	touch $(MJKEY_PATH)
	pip3 install --upgrade pip setuptools
	pip3 install pipenv
	pipenv --python=3.6
	pipenv install dist/garage.tar.gz[all,dev]
	pipenv graph
	# pylint will verify all imports work
	pipenv run pylint --disable=all --enable=import-error garage

ci-deploy-docker: assert-travis
	echo "${DOCKER_API_KEY}" | docker login -u "${DOCKER_USERNAME}" \
		--password-stdin
	docker tag "${TAG}" rlworkgroup/garage-ci:latest
	docker push rlworkgroup/garage-ci

build-ci: TAG ?= rlworkgroup/garage-ci:latest
build-ci: assert-docker-version docker/Dockerfile
	docker build \
		--cache-from rlworkgroup/garage-ci:latest \
		-f docker/Dockerfile \
		--target garage-dev-18.04 \
		-t ${TAG} \
		${BUILD_ARGS} .

build-headless: TAG ?= rlworkgroup/garage-headless:latest
build-headless: assert-docker-version docker/Dockerfile
	docker build \
		-f docker/Dockerfile \
		--cache-from rlworkgroup/garage-headless:latest \
		--target garage-dev-18.04 \
		--build-arg user="$(USER)" \
		--build-arg uid="$(shell id -u)" \
		-t ${TAG} \
		${BUILD_ARGS} .

build-nvidia: TAG ?= rlworkgroup/garage-nvidia:latest
build-nvidia: PARENT_IMAGE ?= nvidia/cuda:10.2-runtime-ubuntu18.04
build-nvidia: assert-docker-version docker/Dockerfile
	docker build \
		-f docker/Dockerfile \
		--cache-from rlworkgroup/garage-nvidia:latest \
		--target garage-nvidia-18.04 \
		-t ${TAG} \
		--build-arg user="$(USER)" \
		--build-arg uid="$(shell id -u)" \
		--build-arg PARENT_IMAGE=${PARENT_IMAGE} \
		${BUILD_ARGS} .

run-ci: ## Run the CI Docker container (only used in TravisCI)
run-ci: TAG ?= rlworkgroup/garage-ci
run-ci:
	docker run \
		-e CODECOV_TOKEN \
		-e TRAVIS_BRANCH \
		-e TRAVIS_PULL_REQUEST \
		-e TRAVIS_COMMIT_RANGE \
		-e TRAVIS \
		-e MJKEY \
		-e GARAGE_GH_TOKEN \
		--memory 7500m \
		--memory-swap 7500m \
		${RUN_ARGS} \
		${TAG} ${RUN_CMD}

run-headless: ## Run the Docker container for headless machines
run-headless: CONTAINER_NAME ?= ''
run-headless: user ?= $$USER
run-headless: ensure-data-path-exists build-headless
	docker run \
		-it \
		--rm \
		-v $(DATA_PATH)/$(CONTAINER_NAME):/home/$(user)/code/garage/data \
		-e MJKEY="$$(cat $(MJKEY_PATH))" \
		--name $(CONTAINER_NAME) \
		${RUN_ARGS} \
		rlworkgroup/garage-headless ${RUN_CMD}

run-nvidia: ## Run the Docker container for machines with NVIDIA GPUs
run-nvidia: ## Requires https://github.com/NVIDIA/nvidia-container-runtime and NVIDIA driver 440+
run-nvidia: CONTAINER_NAME ?= ''
run-nvidia: user ?= $$USER
run-nvidia: GPUS ?= "all"
run-nvidia: ensure-data-path-exists build-nvidia
	xhost +local:docker
	docker run \
		-it \
		--rm \
		--gpus $(GPUS) \
		-v /tmp/.X11-unix:/tmp/.X11-unix \
		-v $(DATA_PATH)/$(CONTAINER_NAME):/home/$(user)/code/garage/data \
		-e DISPLAY=$(DISPLAY) \
		-e QT_X11_NO_MITSHM=1 \
		-e MJKEY="$$(cat $(MJKEY_PATH))" \
		--name $(CONTAINER_NAME) \
		${RUN_ARGS} \
		rlworkgroup/garage-nvidia ${RUN_CMD}

run-nvidia-headless: ## Run the Docker container for machines with NVIDIA GPUs
run-nvidia-headless: ## Requires https://github.com/NVIDIA/nvidia-container-runtime and NVIDIA driver 440+
run-nvidia-headless: CONTAINER_NAME ?= ''
run-nvidia-headless: user ?= $$USER
run-nvidia-headless: GPUS ?= "all"
run-nvidia-headless: ensure-data-path-exists build-nvidia
	docker run \
		-it \
		--rm \
		--gpus $(GPUS) \
		-v $(DATA_PATH)/$(CONTAINER_NAME):/home/$(user)/code/garage/data \
		-e DISPLAY=$(DISPLAY) \
		-e QT_X11_NO_MITSHM=1 \
		-e MJKEY="$$(cat $(MJKEY_PATH))" \
		--name $(CONTAINER_NAME) \
		${RUN_ARGS} \
		rlworkgroup/garage-nvidia ${RUN_CMD}

# Checks that we are in a docker container
assert-docker:
	@test -f /proc/1/cgroup && /bin/grep -qa docker /proc/1/cgroup \
		|| (echo 'This recipe is only to be run inside Docker.' && exit 1)

# Checks that we are in a docker container
assert-travis:
ifndef TRAVIS
	@echo 'This recipe is only to be run from TravisCI'
	@exit 1
endif

ensure-data-path-exists:
	mkdir -p $(DATA_PATH)/$(CONTAINER_NAME) || { echo "Cannot create directory $(DATA_PATH)/$(CONTAINER_NAME)"; exit 1; }

# Check that the docker version is 19.03 or higher
assert-docker-version:
	@[[ $(shell docker version -f "{{.Server.Version}}" | cut -d'.' -f 1) > 18 ]] \
		|| { echo "You need docker 19.03 or higher to build garage"; exit 1; }

# Help target
# See https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
help: ## Display this message
help:
	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
		| sort \
		| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
	ARG PARENT_IMAGE=ubuntu:18.04

	# Garage base target ###########################################################
	FROM $PARENT_IMAGE AS garage-18.04

	# http://bugs.python.org/issue19846
	# > At the moment, setting "LANG=C" on a Linux system fundamentally breaks Python 3, and that's not OK.
	ENV LANG C.UTF-8

	# apt dependencies
	RUN \
	apt-get -y -q update && \
	# Prevents debconf from prompting for user input
	# See https://github.com/phusion/baseimage-docker/issues/58
	DEBIAN_FRONTEND=noninteractive apt-get install -y \
	# Dockerfile deps
	wget \
	unzip \
	git \
	curl \
	# For building glfw
	cmake \
	xorg-dev \
	# mujoco_py
	# See https://github.com/openai/mujoco-py/blob/master/Dockerfile
	libglew-dev \
	libosmesa6-dev \
	patchelf \
	# OpenAI baselines
	libopenmpi-dev \
	# virtualenv
	libbz2-dev \
	libreadline-dev \
	libssl-dev \
	libsqlite3-dev \
	# ruby
	ruby-full && \
	apt-get clean && \
	rm -rf /var/lib/apt/lists/*

	# Build GLFW because the Ubuntu 18.04 version is too old
	# See https://github.com/glfw/glfw/issues/1004
	RUN wget https://github.com/glfw/glfw/releases/download/3.3/glfw-3.3.zip && \
	unzip glfw-3.3.zip && \
	rm glfw-3.3.zip && \
	cd glfw-3.3 && \
	mkdir glfw-build && \
	cd glfw-build && \
	cmake -DBUILD_SHARED_LIBS=ON -DGLFW_BUILD_EXAMPLES=OFF -DGLFW_BUILD_TESTS=OFF -DGLFW_BUILD_DOCS=OFF .. && \
	make -j"$(nproc)" && \
	make install && \
	cd ../../ && \
	rm -rf glfw-3.3

	ARG user=garage-user
	ARG uid=999
	RUN groupadd -g $uid $user && \
	useradd -m -r -u $uid -g $user $user && \
	chown -R $user:$user /home/$user
	ENV USER $user
	USER $user
	ENV HOME /home/$user
	ENV PYENV_ROOT "$HOME/.pyenv"
	ENV PATH "$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH:$HOME/.local/bin"
	ENV PATH_NO_VENV $PATH
	WORKDIR $HOME

	RUN curl https://pyenv.run \| bash && \
	pyenv install 3.6.10 && \
	pyenv global 3.6.10 && \
	pip install virtualenv && \
	rm -r $HOME/.cache/pip

	# MuJoCo 2.0 (for dm_control)
	RUN mkdir -p $HOME/.mujoco && \
	wget https://www.roboti.us/download/mujoco200_linux.zip -O mujoco.zip --no-check-certificate && \
	unzip mujoco.zip -d $HOME/.mujoco && \
	rm mujoco.zip && \
	ln -s $HOME/.mujoco/mujoco200_linux $HOME/.mujoco/mujoco200
	ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$HOME/.mujoco/mujoco200/bin

	# Fixes Segmentation Fault
	# See: https://github.com/openai/mujoco-py/pull/145#issuecomment-356938564
	ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libGLEW.so

	# Set MuJoCo rendering mode (for dm_control)
	ENV MUJOCO_GL "glfw"


	# Create virtualenv
	ENV VIRTUAL_ENV $HOME/venv
	RUN python -m virtualenv $VIRTUAL_ENV
	ENV PATH "$VIRTUAL_ENV/bin:$PATH"

	# Prevent pip from complaining about available upgrades inside virtualenv
	RUN pip install --upgrade pip setuptools wheel && \
	rm -r $HOME/.cache/pip

	# We need a MuJoCo key to install mujoco_py
	# In this step only the presence of the file mjkey.txt is required, so we only
	# create an empty file
	RUN touch $HOME/.mujoco/mjkey.txt

	COPY --chown=$user:$user docker/entrypoint-runtime.sh $HOME/code/garage/docker/entrypoint-runtime.sh
	ENTRYPOINT ["code/garage/docker/entrypoint-runtime.sh"]

	# Headless machine with xvfb, i.e. so rendering works ##########################
	FROM garage-18.04 AS garage-headless-18.04
	USER root
	RUN \
	apt-get -y -q update && \
	# Prevents debconf from prompting for user input
	# See https://github.com/phusion/baseimage-docker/issues/58
	DEBIAN_FRONTEND=noninteractive apt-get install -y \
	# Dummy X server
	xvfb \
	pulseaudio && \
	apt-get clean && \
	rm -rf /var/lib/apt/lists/* && \
	mkdir -p -m 1777 /tmp/.X11-unix

	ARG user=garage-user
	ARG uid=999
	USER $user

	# Ready, set, go.
	COPY --chown=$user:$user docker/entrypoint-headless.sh $HOME/code/garage/docker/entrypoint-headless.sh
	ENTRYPOINT ["code/garage/docker/entrypoint-headless.sh"]


	# Special setup for garage developers, which installs garage from source #######
	# Assumes $PWD is the garage repository.
	FROM garage-headless-18.04 AS garage-dev-18.04

	# apt dependencies
	USER root
	RUN \
	apt-get -y -q update && \
	# Prevents debconf from prompting for user input
	# See https://github.com/phusion/baseimage-docker/issues/58
	DEBIAN_FRONTEND=noninteractive apt-get install -y \
	# graphviz for autoapi documentation
	graphviz && \
	apt-get clean && \
	rm -rf /var/lib/apt/lists/*

	ARG user=garage-user
	ARG uid=999
	USER $user

	# Copy over just setup.py first, so the Docker cache doesn't expire until
	# dependencies change
	#
	# Files needed to run setup.py
	# - README.md
	# - VERSION
	# - scripts/garage
	# - src/garage/__init__.py
	# - setup.py
	COPY --chown=$user:$user README.md $HOME/code/garage/README.md
	COPY --chown=$user:$user VERSION $HOME/code/garage/VERSION
	COPY --chown=$user:$user scripts/garage $HOME/code/garage/scripts/garage
	COPY --chown=$user:$user src/garage/__init__.py $HOME/code/garage/src/garage/__init__.py
	COPY --chown=$user:$user setup.py $HOME/code/garage/setup.py
	WORKDIR $HOME/code/garage

	# Pre-build pre-commit env
	COPY --chown=$user:$user .pre-commit-config.yaml $HOME/code/garage
	RUN git init && \
	pip install pre-commit && \
	pre-commit install && \
	pre-commit install-hooks && \
	rm -r $HOME/.cache/pip

	# Install deps (but not the code)
	RUN pip install --upgrade pip setuptools wheel && \
	pip install .[all,dev] && \
	rm $HOME/.mujoco/mjkey.txt && \
	rm -r $HOME/.cache/pip

	# Add code stub last (ensures code changes have the shortest builds)
	COPY --chown=$user:$user . $HOME/code/garage

	# Build and install the sdist
	RUN python setup.py sdist && \
	cp dist/*.tar.gz dist/garage.tar.gz && \
	pip install dist/garage.tar.gz[all,dev]

	RUN cd benchmarks && python setup.py sdist && \
	cp dist/*.tar.gz dist/benchmarks.tar.gz && \
	pip install dist/benchmarks.tar.gz

	ENTRYPOINT ["docker/entrypoint-headless.sh"]

	# Nvidia target ################################################################
	FROM garage-dev-18.04 AS garage-nvidia-18.04
	ENTRYPOINT ["docker/entrypoint-runtime.sh"]

	# Test target ##################################################################
	FROM garage-dev-18.04 AS garage-test-18.04
	CMD nice -n 11 pytest -v -n auto -m 'not huge and not flaky' --durations=20
	SHELL := /bin/bash

	.PHONY: help test check docs ci-job-normal ci-job-large ci-job-nightly \
	ci-job-verify-envs ci-verify-envs-conda ci-verify-envs-pipenv build-ci \
	build-headless build-nvidia run-ci run-headless run-nvidia assert-docker

	.DEFAULT_GOAL := help

	# Path in host where the experiment data obtained in the container is stored
	DATA_PATH ?= $(shell pwd)/data
	# Set the environment variable MJKEY with the contents of the file specified by
	# MJKEY_PATH.
	MJKEY_PATH ?= ${HOME}/.mujoco/mjkey.txt


	build-test: TAG ?= rlworkgroup/garage-test
	build-test: assert-docker-version docker/Dockerfile
	docker build \
	-f docker/Dockerfile \
	--cache-from rlworkgroup/garage-test:latest \
	--cache-from rlworkgroup/garage-headless:latest \
	--target garage-test-18.04 \
	-t ${TAG} \
	${BUILD_ARGS} \
	.

	test: ## Run the garage-test-18.04 docker target
	test: RUN_ARGS = --memory 7500m --memory-swap 7500m
	test: TAG ?= rlworkgroup/garage-test
	test: CONTAINER_NAME ?= ''
	test: build-test
	@echo "Running test suite..."
	docker run \
	-it \
	--rm \
	-e MJKEY="$$(cat $(MJKEY_PATH))" \
	--name ${CONTAINER_NAME} \
	${RUN_ARGS} \
	${TAG}

	docs: ## Build HTML documentation
	docs:
	@pushd docs && make html && popd
	@python -c 'import os, webbrowser; webbrowser.open("file://" + os.path.realpath("docs/_build/html/index.html"))'

	ci-job-precommit: assert-docker
	scripts/travisci/check_precommit.sh
	@pushd docs && make doctest clean && popd

	ci-job-normal: assert-docker
	[ ! -f $(MJKEY_PATH) ] \|\| mv $(MJKEY_PATH) $(MJKEY_PATH).bak
	pytest --cov=garage --cov-report=xml -m \
	'not nightly and not huge and not flaky and not large and not mujoco and not mujoco_long' --durations=20
	for i in {1..5}; do \
	bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
	\|\| echo 'Retrying...' && sleep 30 && continue; \
	exit 1; \
	done

	ci-job-large: assert-docker
	[ ! -f $(MJKEY_PATH) ] \|\| mv $(MJKEY_PATH) $(MJKEY_PATH).bak
	pytest --cov=garage --cov-report=xml -m 'large and not flaky' --durations=20
	for i in {1..5}; do \
	bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
	\|\| echo 'Retrying...' && sleep 30 && continue; \
	exit 1; \
	done

	ci-job-mujoco: assert-docker
	pytest --cov=garage --cov-report=xml -m 'mujoco and not flaky' --durations=20
	for i in {1..5}; do \
	bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
	\|\| echo 'Retrying...' && sleep 30 && continue; \
	exit 1; \
	done

	ci-job-mujoco-long: assert-docker
	pytest --cov=garage --cov-report=xml -m 'mujoco_long and not flaky' --durations=20
	for i in {1..5}; do \
	bash <(curl -s https://codecov.io/bash --retry 5) -Z && break \
	\|\| echo 'Retrying...' && sleep 30 && continue; \
	exit 1; \
	done

	ci-job-nightly: assert-docker
	pytest -m nightly

	ci-job-verify-envs: assert-docker ci-job-verify-envs-pipenv ci-job-verify-envs-conda

	ci-job-verify-envs-conda: assert-docker
	ci-job-verify-envs-conda: CONDA_ROOT := $$HOME/miniconda
	ci-job-verify-envs-conda: CONDA := $(CONDA_ROOT)/bin/conda
	ci-job-verify-envs-conda: GARAGE_BIN = $(CONDA_ROOT)/envs/garage-ci/bin
	ci-job-verify-envs-conda:
	touch $(MJKEY_PATH)
	wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
	bash miniconda.sh -b -p $(CONDA_ROOT)
	hash -r
	$(CONDA) config --set always_yes yes --set changeps1 no
	# Related issue: https://github.com/conda/conda/issues/9105
	# Fix in conda: https://github.com/conda/conda/pull/9014
	# https://repo.continuum.io/miniconda/ doesn't have the script for 4.7.12 yet,
	# so CI fetches 4.7.10 and runs into the above issue when trying to update conda
	$(CONDA) install -c anaconda setuptools
	$(CONDA) update -q conda
	$(CONDA) init
	# Useful for debugging any issues with conda
	$(CONDA) info -a
	$(CONDA) create -n garage-ci python=3.6 pip -y
	$(GARAGE_BIN)/pip install --upgrade pip setuptools
	$(GARAGE_BIN)/pip install dist/garage.tar.gz[all,dev]
	# pylint will verify all imports work
	$(GARAGE_BIN)/pylint --disable=all --enable=import-error garage

	# The following two lines remove the Dockerfile's built-in virtualenv from the
	# path, so we can test with pipenv directly
	ci-job-verify-envs-pipenv: assert-docker
	ci-job-verify-envs-pipenv: export PATH=$(shell echo $$PATH_NO_VENV)
	ci-job-verify-envs-pipenv: export VIRTUAL_ENV=
	ci-job-verify-envs-pipenv: export PIPENV_MAX_RETRIES=2 # number of retries for network requests. Default is 0
	ci-job-verify-envs-pipenv:
	touch $(MJKEY_PATH)
	pip3 install --upgrade pip setuptools
	pip3 install pipenv
	pipenv --python=3.6
	pipenv install dist/garage.tar.gz[all,dev]
	pipenv graph
	# pylint will verify all imports work
	pipenv run pylint --disable=all --enable=import-error garage

	ci-deploy-docker: assert-travis
	echo "${DOCKER_API_KEY}" \| docker login -u "${DOCKER_USERNAME}" \
	--password-stdin
	docker tag "${TAG}" rlworkgroup/garage-ci:latest
	docker push rlworkgroup/garage-ci

	build-ci: TAG ?= rlworkgroup/garage-ci:latest
	build-ci: assert-docker-version docker/Dockerfile
	docker build \
	--cache-from rlworkgroup/garage-ci:latest \
	-f docker/Dockerfile \
	--target garage-dev-18.04 \
	-t ${TAG} \
	${BUILD_ARGS} .

	build-headless: TAG ?= rlworkgroup/garage-headless:latest
	build-headless: assert-docker-version docker/Dockerfile
	docker build \
	-f docker/Dockerfile \
	--cache-from rlworkgroup/garage-headless:latest \
	--target garage-dev-18.04 \
	--build-arg user="$(USER)" \
	--build-arg uid="$(shell id -u)" \
	-t ${TAG} \
	${BUILD_ARGS} .

	build-nvidia: TAG ?= rlworkgroup/garage-nvidia:latest
	build-nvidia: PARENT_IMAGE ?= nvidia/cuda:10.2-runtime-ubuntu18.04
	build-nvidia: assert-docker-version docker/Dockerfile
	docker build \
	-f docker/Dockerfile \
	--cache-from rlworkgroup/garage-nvidia:latest \
	--target garage-nvidia-18.04 \
	-t ${TAG} \
	--build-arg user="$(USER)" \
	--build-arg uid="$(shell id -u)" \
	--build-arg PARENT_IMAGE=${PARENT_IMAGE} \
	${BUILD_ARGS} .

	run-ci: ## Run the CI Docker container (only used in TravisCI)
	run-ci: TAG ?= rlworkgroup/garage-ci
	run-ci:
	docker run \
	-e CODECOV_TOKEN \
	-e TRAVIS_BRANCH \
	-e TRAVIS_PULL_REQUEST \
	-e TRAVIS_COMMIT_RANGE \
	-e TRAVIS \
	-e MJKEY \
	-e GARAGE_GH_TOKEN \
	--memory 7500m \
	--memory-swap 7500m \
	${RUN_ARGS} \
	${TAG} ${RUN_CMD}

	run-headless: ## Run the Docker container for headless machines
	run-headless: CONTAINER_NAME ?= ''
	run-headless: user ?= $$USER
	run-headless: ensure-data-path-exists build-headless
	docker run \
	-it \
	--rm \
	-v $(DATA_PATH)/$(CONTAINER_NAME):/home/$(user)/code/garage/data \
	-e MJKEY="$$(cat $(MJKEY_PATH))" \
	--name $(CONTAINER_NAME) \
	${RUN_ARGS} \
	rlworkgroup/garage-headless ${RUN_CMD}

	run-nvidia: ## Run the Docker container for machines with NVIDIA GPUs
	run-nvidia: ## Requires https://github.com/NVIDIA/nvidia-container-runtime and NVIDIA driver 440+
	run-nvidia: CONTAINER_NAME ?= ''
	run-nvidia: user ?= $$USER
	run-nvidia: GPUS ?= "all"
	run-nvidia: ensure-data-path-exists build-nvidia
	xhost +local:docker
	docker run \
	-it \
	--rm \
	--gpus $(GPUS) \
	-v /tmp/.X11-unix:/tmp/.X11-unix \
	-v $(DATA_PATH)/$(CONTAINER_NAME):/home/$(user)/code/garage/data \
	-e DISPLAY=$(DISPLAY) \
	-e QT_X11_NO_MITSHM=1 \
	-e MJKEY="$$(cat $(MJKEY_PATH))" \
	--name $(CONTAINER_NAME) \
	${RUN_ARGS} \
	rlworkgroup/garage-nvidia ${RUN_CMD}

	run-nvidia-headless: ## Run the Docker container for machines with NVIDIA GPUs
	run-nvidia-headless: ## Requires https://github.com/NVIDIA/nvidia-container-runtime and NVIDIA driver 440+
	run-nvidia-headless: CONTAINER_NAME ?= ''
	run-nvidia-headless: user ?= $$USER
	run-nvidia-headless: GPUS ?= "all"
	run-nvidia-headless: ensure-data-path-exists build-nvidia
	docker run \
	-it \
	--rm \
	--gpus $(GPUS) \
	-v $(DATA_PATH)/$(CONTAINER_NAME):/home/$(user)/code/garage/data \
	-e DISPLAY=$(DISPLAY) \
	-e QT_X11_NO_MITSHM=1 \
	-e MJKEY="$$(cat $(MJKEY_PATH))" \
	--name $(CONTAINER_NAME) \
	${RUN_ARGS} \
	rlworkgroup/garage-nvidia ${RUN_CMD}

	# Checks that we are in a docker container
	assert-docker:
	@test -f /proc/1/cgroup && /bin/grep -qa docker /proc/1/cgroup \
	\|\| (echo 'This recipe is only to be run inside Docker.' && exit 1)

	# Checks that we are in a docker container
	assert-travis:
	ifndef TRAVIS
	@echo 'This recipe is only to be run from TravisCI'
	@exit 1
	endif

	ensure-data-path-exists:
	mkdir -p $(DATA_PATH)/$(CONTAINER_NAME) \|\| { echo "Cannot create directory $(DATA_PATH)/$(CONTAINER_NAME)"; exit 1; }

	# Check that the docker version is 19.03 or higher
	assert-docker-version:
	@[[ $(shell docker version -f "{{.Server.Version}}" \| cut -d'.' -f 1) > 18 ]] \
	\|\| { echo "You need docker 19.03 or higher to build garage"; exit 1; }

	# Help target
	# See https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
	help: ## Display this message
	help:
	@grep -E '^[a-zA-Z_-]+:.?## .$$' $(MAKEFILE_LIST) \
	\| sort \
	\| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'