Speed up Docker builds on TravisCI by reusing dependency containers

.travis.yml

language: python
python:
  - "2.7"
sudo: required
services:
  - docker
install:
  - $TRAVIS_BUILD_DIR/build-containers.py
script:
  - docker run --rm my-app test

build-containers.py

#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
import subprocess
import sys
import hashlib

DOCKER_REPO = 'quay.io/namespace/my-app:%s'

def main():
    try:
        build_dir = os.getenv('TRAVIS_BUILD_DIR', os.getcwd())

        # Calculate the dependencies hash by concatting Dockerfile.deps and requirements.txt
        dep_sha = hashlib.sha1()
        for file_name in ['Dockerfile.deps', 'requirements.txt']:
            with open(os.path.join(build_dir, file_name), 'r') as fp:
                dep_sha.update(fp.read())
        dep_sha = dep_sha.hexdigest()[:7]
        deps_image = DOCKER_REPO % ('deps-%s' % (dep_sha,),)

        # Attempt to download the prebuilt image
        try:
            run_command(['docker', 'pull', deps_image])
            prebuilt_deps = True
        except subprocess.CalledProcessError:
            prebuilt_deps = False

        if prebuilt_deps:
            # Make sure that the prebuilt image is also available as my-app-deps
            run_command(['docker', 'tag', deps_image, 'my-app-deps'])
        else:
            # Build the dependencies image
            run_command(['docker', 'build', '--force-rm', '-t', 'my-app-deps', '-f', 'Dockerfile.deps', build_dir])

            # Tag & push the new dependencies image
            run_command(['docker', 'tag', 'my-app-deps', deps_image])
            run_command(['docker', 'push', deps_image])

        # Build the actual image
        run_command(['docker', 'build', '--force-rm', '-t', 'my-app', '-f', 'Dockerfile', build_dir])

        # Untag the intermediate build images
        run_command(['docker', 'rmi', 'my-app-deps', deps_image])

    except Exception as e:
        print('E: %s' % e)
        print('')
        sys.exit(1)

def run_command(args):
    print('> %s' % (' '.join(args),))

    proc = subprocess.Popen(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
    retcode = proc.wait()

    if retcode:
        cmd = args[0]
        raise subprocess.CalledProcessError(retcode, cmd)

if __name__ == "__main__":
    main()

Dockerfile

# my-app-deps is a pseudo container which can be built from Dockerfile.deps
FROM my-app-deps

# Copy over the app
COPY . /usr/src/app

Dockerfile.deps

FROM python:2.7

RUN mkdir -p /usr/src/app
WORKDIR /usr/src/app

# Download/build dependencies only
COPY requirements.txt /usr/src/app/
RUN pip install --no-cache-dir -r requirements.txt