Docker example

.dockerignore

.git
.ipynb_checkpoints
.tox
*.ipynb
notebooks
*.egg-info
*.pyc
OLD-tf2-conversion
temp_data
a_really_large_folder_that_is_not_needed_to_be_in_the_docker_image
my_project/data_loaders/__pycache__
my_project/collect_data/__pycache__
my_project/models/__pycache__
my_project/__pycache__

build-docker-image.sh

#!/bin/bash

set -eux

STAGE=${1-test}
ARCH=${2-gpu}

DIR=$(dirname "$0")
pushd $DIR/..
trap popd EXIT

if [[ $ARCH == 'gpu' ]]; then
    IMAGE=some_url/my_project:tf2.0.0-py3
    DOCKERFILE=scripts/Dockerfile
elif [[ $ARCH == 'cpu' ]]; then
    IMAGE=some_url/my_project-cpu:tf2.0.0-py3
    DOCKERFILE=scripts/Dockerfile.cpu
fi

docker build --tag $IMAGE --file $DOCKERFILE .

if [[ $STAGE == 'release' ]]; then
    docker push $IMAGE
fi

docker-train.sh

#!/bin/bash -eux

WHERE=${1-remote}
INTERACTIVE=${2-noninteractive}

CONTAINER_NAME=docker-train

DIR=$(dirname $0)

#IMAGE=tensorflow/tensorflow:2.0.0-gpu-py3
IMAGE=some_url/my_project:tf2.0.0-py3
docker pull $IMAGE

docker kill $CONTAINER_NAME 2>/dev/null || true
docker rm $CONTAINER_NAME 2>/dev/null || true


docker run \
       --runtime=nvidia \
       --restart=on-failure:20 \
       -e PYTHONPATH=/my_project \
       -v $HOME/my_project:/my_project \
       --name $CONTAINER_NAME \
       -e DOCKER_HOST=$(hostname) \
       $IMAGE \
       python my_project_source/train.py

Dockerfile

FROM tensorflow/tensorflow:2.0.0-gpu-py3

RUN apt-get update && apt-get install -y --no-install-recommends \
        git \
        libsamplerate0-dev \
        libsndfile1 \
        ffmpeg \
        curl \
        zlib1g-dev \
        libzip-dev \

RUN pip install --upgrade pip

COPY . /my_project
WORKDIR /my_project

RUN pip install .[gpu]

project folder structure

my_project
  - my_project_source
    - train.py
  - a_really_large_folder_that_is_not_needed_to_be_in_the_docker_image
  - temp_data
  - notebooks
    - something.ipynb
  - scripts
    - Dockerfile
    - Dockerfile.cpu
    - build-docker-image.sh
    - docker-train.sh
  my_project.egg-info
  setup.py
  .dockerignore
  

setup.py

from setuptools import setup, find_packages

setup(
    name='my_project',
    description='world piece, etc',
    version='0.0.1',
    packages=find_packages(exclude=['test', '*.test', '*.test.*']),
    include_package_data=True,
    install_requires=[
        'numpy==1.17.2',
    ],
    extras_require={
        'cpu': [
            'tensorflow==2.0.0',
        ],
        'gpu': [
            'tensorflow-gpu==2.0.0',
        ],
    },

)