Concurrent map write on cache export in Docker / Moby / Buildkit

README.md

Concurrent map write on cache export in Docker / Moby / Buildkit

This is an example of Dockerfile + compose that generates race condition in Docker daemon with fatal error that breaks it completely.

moby/buildkit#2041

1. Save all files locally
2. Configure REGISTRY environment variable in build-and-push-caches.sh to yours remote container image registry.
3. Start build-and-push-caches.sh.
4. Start wait-for-docker-failure.sh.
5. Wait for failure.
6. PROFIT!!!

build-and-push-caches.sh

#!/bin/sh

# Build and push images with inline cache to remote registry

set -e

export REGISTRY=my-sooper-dooper-registry.example.com/
export BUILDKIT_INLINE_CACHE=1

# Build and push only first service at first
docker compose build --push service-0

# Build and push all services
docker compose build --push

docker-compose.yaml

---
x-cache-from:
  build: &x-cache-from
    cache_from:
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-0:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-1:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-2:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-3:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-4:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-5:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-6:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-7:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-8:latest"
      - "${REGISTRY:-}test-docker-multiple-build-and-push-service-9:latest"
x-args: &x-args
  REBUILD_SEED: "${REBUILD_SEED:-0}"
  SIZE: "${SIZE:-1M}"
  BUILDKIT_INLINE_CACHE: "${BUILDKIT_INLINE_CACHE:-0}"

services:
  service-0:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '0'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-0:latest"
  service-1:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '1'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-1:latest"
  service-2:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '2'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-2:latest"
  service-3:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '3'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-3:latest"
  service-4:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '4'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-4:latest"
  service-5:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '5'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-5:latest"
  service-6:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '6'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-6:latest"
  service-7:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '7'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-7:latest"
  service-8:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '8'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-8:latest"
  service-9:
    build:
      <<: *x-cache-from
      args:
        <<: *x-args
        SEED: '9'
      context: .
    image: "${REGISTRY:-}test-docker-multiple-build-and-push-service-9:latest"

Dockerfile

FROM ubuntu

ARG REBUILD_SEED=0
ARG SIZE=1M

# Common set of layers
RUN head -c "${SIZE}" </dev/urandom >/random00
RUN head -c "${SIZE}" </dev/urandom >/random01
RUN head -c "${SIZE}" </dev/urandom >/random02
RUN head -c "${SIZE}" </dev/urandom >/random03
RUN head -c "${SIZE}" </dev/urandom >/random04
RUN head -c "${SIZE}" </dev/urandom >/random05
RUN head -c "${SIZE}" </dev/urandom >/random06
RUN head -c "${SIZE}" </dev/urandom >/random07
RUN head -c "${SIZE}" </dev/urandom >/random08
RUN head -c "${SIZE}" </dev/urandom >/random09

ARG SEED=0

# Specific layer for each service
RUN head -c "${SIZE}" </dev/urandom >/random10

wait-for-docker-failure.sh

#!/bin/sh

# Start building images in a loop and wait for Docker daemon failure

set -e

while docker compose build; do true; done