Skip to content

Instantly share code, notes, and snippets.

@amodig
Created February 1, 2018 16:04
Show Gist options
  • Save amodig/f27d9727f7a5a2ed4e996ad68d84ff3c to your computer and use it in GitHub Desktop.
Save amodig/f27d9727f7a5a2ed4e996ad68d84ff3c to your computer and use it in GitHub Desktop.
docker-airflow
FROM debian:9
MAINTAINER Arttu Modig <arttu.modig@gmail.com>
# Never prompts the user for choices on installation/configuration of packages
ENV DEBIAN_FRONTEND noninteractive
ENV TERM linux
ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LC_CTYPE en_US.UTF-8
ENV LC_MESSAGES en_US.UTF-8
ENV CONDA_INSTALLER="Miniconda3-latest-Linux-x86_64.sh"
ENV CONDA_APP_USER="airflow"
ENV CONDA_ENV_NAME="airflow"
ENV PATH=/opt/conda/bin:$PATH
# ARG AIRFLOW_VERSION=1.9.0
ARG AIRFLOW_HOME=/usr/local/airflow
# trying to create a minimal debian
RUN apt-get update -qq --fix-missing && \
apt-get install -qq --no-install-recommends \
bzip2 ca-certificates curl dpkg locales grep netcat rsync sed wget \
libglib2.0-0 libxext6 libsm6 libxrender1 \
libkrb5-dev libsasl2-dev libssl-dev libffi-dev \
build-essential libblas-dev liblapack-dev libpq-dev \
git mercurial subversion
# set locale (first create empty locale.gen file)
RUN locale-gen && \
sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen && \
locale-gen && \
update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
# install a minimal conda
RUN wget -q "https://repo.continuum.io/miniconda/${CONDA_INSTALLER}" -O /tmp/miniconda.sh && \
/bin/bash /tmp/miniconda.sh -b -p /opt/conda && \
/opt/conda/bin/conda install --yes conda && \
conda update conda && \
conda install conda-build && \
conda remove tk --yes && \
conda clean --yes --all
# cleaning
RUN apt-get autoremove -qq && \
apt-get clean -qq && \
rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
/tmp/* /var/tmp/* && \
find /opt/conda \
\( -type d -a -name test -o -name tests \) \
-o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \
-exec rm -rf '{}' +
# reduce image size
RUN for x in `ls /usr/share/locale | grep -v -i en | grep -v -i local`; \
do rm -fr /usr/share/locale/$x; \
done && \
rm -fr /usr/share/locale/ca* \
/usr/share/locale/den \
/usr/share/locale/men \
/usr/share/locale/wen \
/usr/share/locale/zen && \
rm -fr /usr/share/doc/* /usr/share/doc-base/* /usr/share/man/* \
/usr/share/groff/* /usr/share/info/* \
/usr/share/lintian/* /usr/share/linda/* /var/cache/man/*
# Add the user that will run the app (no need to run as root)
RUN groupadd -r ${CONDA_APP_USER} && \
useradd -mr -g ${CONDA_APP_USER} -s /bin/bash -d ${AIRFLOW_HOME} ${CONDA_APP_USER}
# Install and configure airflow conda environment
COPY environment.yml ${AIRFLOW_HOME}/environment.yml
RUN conda config --append channels conda-forge && \
conda env create -n ${CONDA_ENV_NAME} -f ${AIRFLOW_HOME}/environment.yml && \
conda clean --yes --all
COPY dags ${AIRFLOW_HOME}/dags
COPY script/entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
RUN chown -R ${CONDA_APP_USER}:${CONDA_APP_USER} ${AIRFLOW_HOME}
USER ${CONDA_APP_USER}
WORKDIR ${AIRFLOW_HOME}
# activate the conda environment
ENV PATH /opt/conda/envs/${CONDA_ENV_NAME}/bin:$PATH
# webserver; Celery flower; worker log
EXPOSE 8080 5555 8793
ENTRYPOINT ["/entrypoint.sh"]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment