pip installing a private python package (shimage, in this case) into a docker image via docker buildkit

command

eval $(ssh-agent)
ssh-add ~/.ssh/id_ed25519
DOCKER_BUILDKIT=1 docker build -f Dockerfile.doesnt  --ssh default=${SSH_AUTH_SOCK} . -t shimmed-airflow

Dockerfile

FROM docker.io/apache/airflow:2.5.0

USER root

# install python (not depending on the system python for reasons...)
COPY --from=python:3.11-slim /usr/local/bin/*3.11* /usr/local/bin/
COPY --from=python:3.11-slim /usr/local/lib/pkgconfig/*3.11* /usr/local/pkgconfig/
COPY --from=python:3.11-slim /usr/local/lib/*3.11*.so  /usr/local/lib/
COPY --from=python:3.11-slim /usr/local/lib/python3.11  /usr/local/lib/python3.11
RUN ldconfig

# prepare shimage venv
RUN apt update && apt install -y git openssh-client && rm -rf /var/lib/apt/lists/*
RUN mkdir -p /opt
RUN touch /shimage-init.sh
ENV PIP_USER=no
RUN python3.11 -m venv /opt/shimage
RUN echo 'source /opt/shimage/bin/activate' >> /shimage-init.sh
RUN bash -c 'source /shimage-init.sh && pip install --upgrade pip wheel'

# install shimage
RUN mkdir -m 700 ~/.ssh
RUN ssh-keyscan github.com > ~/.ssh/known_hosts
RUN --mount=type=ssh bash -c "source /shimage-init.sh && pip install 'git+ssh://git@github.com/astronomer/shimage'"
RUN chown -R airflow /opt/shimage 

USER airflow

# shim airflow
RUN bash -c 'source /shimage-init.sh && shimage procio mkshim airflow'
RUN bash -c "source /shimage-init.sh && shimage fswrites mkshim airflow --check $AIRFLOW_HOME'/*'"