building ceph in containers

Dockerfile

FROM quay.io/ceph-ci/ceph:master

ADD ceph.spec.in /bootstrap/ceph.spec.in
ADD install-deps.sh /bootstrap/install-deps.sh

# ceph repo for the installed sha1 may no longer exist - basing this on a plain centos container would probably make more sense
RUN dnf config-manager --set-disabled Ceph Ceph-noarch ceph-source
RUN dnf install -y git virtualenv
RUN /bin/bash -c 'cd /bootstrap && ./install-deps.sh'
RUN rm -rf /bootstrap

prepare-dev-container.sh

#!/bin/bash

VERBOSE=${VERBOSE:-0}
DEVDIR=/ceph

echo "Listing files in installed packages..."
dnf -y install yum-utils

RELEASE=$(repoquery -C --qf '%{release}' ceph-common)
echo "Found ceph release $RELEASE"
PKGS=$(dnf -C list --installed | grep $RELEASE | awk '{print $1}')
FILES=""
for PKG in $PKGS
do
    FILES+=$(repoquery -C -l $PKG)$'\n'
done

echo "Removing .pyc files..."
# remove pyc FILES
for f in $(echo "$FILES" | grep '\.pyc')
do
    rm -f "$f"
done

EXCLUDES=$(cat <<EOF
.*__pycache__.*
.*.pyc
.*egg-info.*
/usr/lib/.build-id
/usr/lib/systemd
/usr/lib/tmpfiles.d
/usr/sbin/ceph-volume.*
/usr/share/man
/usr/share/doc
/usr/share
/etc/ceph/rbdmap
/var/lib/cephadm/.ssh/authorized_keys
EOF
)

echo "$FILES" > after_excludes

while IFS= read -r PATTERN
do
    FILES=$(echo "$FILES" | grep -v "$PATTERN" | sort)
done <<< "$EXCLUDES"

# since we remove FILES from the list as we go with grep, prefixes
# that overlap must be in order from more-specific to less-specific
REMAP_STATIC=$(cat <<EOF
/etc/bash_completion.d src/bash_completion
/etc/grafana/dashboards/ceph-dashboard monitoring/grafana/dashboards
/etc/logrotate.d/ceph src/logrotate.conf
/etc/prometheus/ceph monitoring/prometheus/alerts
/etc/sudoers.d sudoers.d
/etc/sysconfig/ceph etc/sysconfig/ceph
/usr/lib/ceph/ceph_common.sh src/ceph_common.sh
/usr/lib/ceph/ceph-osd-prestart.sh src/ceph-osd-prestart.sh
/usr/lib/python3.6/site-packages/ceph_argparse.py src/pybind/ceph_argparse.py
/usr/lib/python3.6/site-packages/ceph_daemon.py src/pybind/ceph_daemon.py
/usr/lib/python3.6/site-packages/ceph_volume src/ceph-volume/ceph_volume
/usr/lib/python3.6/site-packages/ceph src/python-common/ceph
/usr/sbin/ceph-create-keys src/ceph-create-keys
/usr/sbin/cephadm src/cephadm/cephadm
/usr/share/mgr src/pybind/mgr
EOF
)

REMAP_PATTERNS=$(cat <<EOF
/usr/bin build/bin
/usr/lib/sysctl.d etc/sysctl
/usr/lib64/python3.6/site-packages.*\.so build/lib/cython_modules/lib.3
/usr/lib64/.*\.so.* build/lib
/usr/sbin/mount.* build/bin
EOF
)

echo "Replacing static list of files and dirs from packages..."

# remap static files first, then patterns
while IFS= read -r LINE
do
    DEST=$(echo "$LINE" | cut -d ' ' -f 1)
    SRC=$(echo "$LINE" | cut -d ' ' -f 2)
    [[ "$VERBOSE" -gt 0 ]] && echo "remapping static $DEST"
    rm -rf "$DEST"
    ln -s "$DEVDIR/$SRC" "$DEST"
    FILES=$(echo "$FILES" | grep -v -F "$DEST")
done <<< "$REMAP_STATIC"

echo "Replacing packaged files based on patterns..."

while IFS= read -r LINE
do
    PATTERN=$(echo "$LINE" | cut -d ' ' -f 1)
    SRC=$(echo "$LINE" | cut -d ' ' -f 2)
    for MATCH in $(echo "$FILES" | grep "$PATTERN")
    do
        [[ "$VERBOSE" -gt 0 ]] && echo "remapping from pattern $PATTERN $MATCH"
        FILENAME=$(basename "$MATCH")
        rm -f "$MATCH"
        ln -s "$DEVDIR/$SRC/$FILENAME" "$MATCH"
    done
    FILES=$(echo "$FILES" | grep -v "$PATTERN")
done <<< "$REMAP_PATTERNS"

# don't worry about directories, just check for any
# files we didn't replace
ERR=0
for f in $(echo $FILES)
do
    test -f $f && echo "WARNING: unhandled file $f" && ERR=1
done

exit $ERR

run-docker.sh

#!/bin/bash

# adapted from @ideepika's tracing dev scripts

NAME="${NAME:-joshd-build-master}"
CEPH="${CEPH:-$HOME/ceph}"
CCACHE="${CCACHE:-$HOME/ceph-ccache}"
VERSION="${VERSION:-master}"
TAG="${TAG:-test}"

docker stop $NAME && docker rm $NAME

# Creates a container with all recommended configs
docker run -itd \
  -v $CEPH:$CEPH \
  -v $CCACHE:/root/.ccache \
  -v /run/udev:/run/udev:ro \
  -v /slow:/slow \
  --privileged \
  --cap-add=SYS_PTRACE \
  --security-opt seccomp=unconfined \
  --net=host \
  --name=$NAME \
  --hostname=$NAME \
  --add-host=$NAME:127.0.0.1 \
  $TAG 

docker exec -it $NAME /bin/bash

run-prep-container.sh

#!/bin/bash

# This is an alternative approach to the above two files, requiring no
# changes to the container image when redeploying. Instead, we
# replace installed ceph files with symlinks to the src/build tree,
# mounted from the host.

BASE=${BASE:-docker.io/ceph/daemon-base:latest-master-devel}
NAME=${NAME:-$USER-dev}
CEPH=${CEPH:-$HOME/ceph}

sudo podman create -itd --privileged -v $CEPH:$CEPH --name $NAME $BASE
sudo podman cp prepare-dev-container.sh $NAME:/
sudo podman start $NAME
sudo podman exec $NAME /prepare-dev-container.sh/prepare-dev-container.sh
sudo podman exec $NAME ln -s $CEPH /ceph

To build this minimum image, which requires only a couple files, it's faster to put the necessary files in a subdirectory to avoid sending the entire git repo to docker as the build context. e.g.

mkdir dev
cp ceph.spec.in install-deps.sh dev
cd dev
sudo docker build -t joshd-dev-master:v1 -f /path/to/Dockerfile .