docker-airflow

Dockerfile

FROM debian:9
MAINTAINER Arttu Modig <arttu.modig@gmail.com>

# Never prompts the user for choices on installation/configuration of packages
ENV DEBIAN_FRONTEND noninteractive
ENV TERM linux

ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LC_CTYPE en_US.UTF-8
ENV LC_MESSAGES en_US.UTF-8

ENV CONDA_INSTALLER="Miniconda3-latest-Linux-x86_64.sh"
ENV CONDA_APP_USER="airflow"
ENV CONDA_ENV_NAME="airflow"

ENV PATH=/opt/conda/bin:$PATH

# ARG AIRFLOW_VERSION=1.9.0
ARG AIRFLOW_HOME=/usr/local/airflow

# trying to create a minimal debian
RUN apt-get update -qq --fix-missing && \
    apt-get install -qq --no-install-recommends \
    bzip2 ca-certificates curl dpkg locales grep netcat rsync sed wget \
    libglib2.0-0 libxext6 libsm6 libxrender1 \
    libkrb5-dev libsasl2-dev libssl-dev libffi-dev \
    build-essential libblas-dev liblapack-dev libpq-dev \
    git mercurial subversion

# set locale (first create empty locale.gen file)
RUN locale-gen && \
    sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen && \
    locale-gen && \
    update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8

# install a minimal conda
RUN wget -q "https://repo.continuum.io/miniconda/${CONDA_INSTALLER}" -O /tmp/miniconda.sh && \
    /bin/bash /tmp/miniconda.sh -b -p /opt/conda && \
    /opt/conda/bin/conda install --yes conda && \
    conda update conda && \
    conda install conda-build && \
    conda remove tk --yes && \
    conda clean --yes --all

# cleaning
RUN apt-get autoremove -qq && \
    apt-get clean -qq && \
    rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
      /tmp/* /var/tmp/* && \
    find /opt/conda \
      \( -type d -a -name test -o -name tests \) \
      -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \
      -exec rm -rf '{}' +

# reduce image size
RUN for x in `ls /usr/share/locale | grep -v -i en | grep -v -i local`; \
      do rm -fr /usr/share/locale/$x; \
    done && \
    rm -fr /usr/share/locale/ca* \
        /usr/share/locale/den \
        /usr/share/locale/men \
        /usr/share/locale/wen \
        /usr/share/locale/zen && \
    rm -fr /usr/share/doc/* /usr/share/doc-base/* /usr/share/man/* \
        /usr/share/groff/* /usr/share/info/* \
        /usr/share/lintian/* /usr/share/linda/* /var/cache/man/*

# Add the user that will run the app (no need to run as root)
RUN groupadd -r ${CONDA_APP_USER} && \
    useradd -mr -g ${CONDA_APP_USER} -s /bin/bash -d ${AIRFLOW_HOME} ${CONDA_APP_USER}

# Install and configure airflow conda environment
COPY environment.yml ${AIRFLOW_HOME}/environment.yml
RUN conda config --append channels conda-forge && \
    conda env create -n ${CONDA_ENV_NAME} -f ${AIRFLOW_HOME}/environment.yml && \
    conda clean --yes --all
COPY dags ${AIRFLOW_HOME}/dags
COPY script/entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
RUN chown -R ${CONDA_APP_USER}:${CONDA_APP_USER} ${AIRFLOW_HOME}
USER ${CONDA_APP_USER}
WORKDIR ${AIRFLOW_HOME}
# activate the conda environment
ENV PATH /opt/conda/envs/${CONDA_ENV_NAME}/bin:$PATH

# webserver; Celery flower; worker log
EXPOSE 8080 5555 8793

ENTRYPOINT ["/entrypoint.sh"]