Skip to content

Instantly share code, notes, and snippets.

@sebradloff
Created April 17, 2018 21:19
Show Gist options
  • Star 6 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sebradloff/f158874e615bda0005c6f4577b20036e to your computer and use it in GitHub Desktop.
Save sebradloff/f158874e615bda0005c6f4577b20036e to your computer and use it in GitHub Desktop.
Airflow MesosExecutor dockerized workflow
# Mesos specific block configuration
[mesos]
# Mesos master address which MesosExecutor will connect to.
master = localhost:5050
# The framework name which Airflow scheduler will register itself as on mesos
framework_name = Airflow
# Number of cpu cores required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_cpu = 1
# Memory in MB required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_memory = 256
# Enable framework checkpointing for mesos
# See http://mesos.apache.org/documentation/latest/slave-recovery/
checkpoint = False
# Enable framework authentication for mesos
# See http://mesos.apache.org/documentation/latest/configuration/
authenticate = False
# Mesos credentials, if authentication is enabled
# default_principal = admin
# default_secret = admin
# Name of the docker image used for each airflow task
# image created from Dockerfile-airflow build in this case
docker_image_slave = slicelife/docker-airflow
# REFERENCES:
# - https://github.com/puckel/docker-airflow
# - https://github.com/ImDarrenG/mesos-framework-dev/blob/master/Dockerfile
# - https://github.com/Stibbons/docker-airflow-mesos
# Wherever you store your mesos image built from Dockerfile-mesos
FROM slicelife/mesos:1.4.0 as mesos
FROM ubuntu:16.04
# Never prompts the user for choices on installation/configuration of packages
ENV DEBIAN_FRONTEND noninteractive
ENV TERM linux
# Airflow
ARG AIRFLOW_VERSION=1.9.0
ARG AIRFLOW_HOME=/usr/local/airflow
# Define en_US.
ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LC_CTYPE en_US.UTF-8
ENV LC_MESSAGES en_US.UTF-8
ENV LC_ALL en_US.UTF-8
# Mesos
ARG MESOS_VERSION=1.4.0
ARG MESOS_HOME=/usr/local/mesos
ARG MESOS_PYTHON_PACKAGES=${MESOS_HOME}/mesos-${MESOS_VERSION}/src/python/dist/
# Copy Mesos python eggs from upstream Mesos image
RUN mkdir -p ${MESOS_PYTHON_PACKAGES}
COPY --from=mesos ${MESOS_PYTHON_PACKAGES} ${MESOS_PYTHON_PACKAGES}
# Install Mesos python eggs
RUN find ${MESOS_HOME} -name "*.egg"
RUN apt-get update -yq --fix-missing
RUN apt-get install -yq \
python-setuptools \
build-essential \
python-dev \
python-six \
python-virtualenv \
libcurl4-nss-dev \
libsasl2-dev \
libsasl2-modules \
maven \
libapr1-dev \
libsvn-dev \
zlib1g-dev
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.interface-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.executor-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.scheduler-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.native-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos.cli-*.egg
RUN cd ${MESOS_PYTHON_PACKAGES} && easy_install mesos-*.egg
# Install airflow dependencies
RUN set -ex \
&& buildDeps=' \
python3-dev \
libkrb5-dev \
libsasl2-dev \
libssl-dev \
libffi-dev \
build-essential \
libblas-dev \
liblapack-dev \
libpq-dev \
git \
' \
&& apt-get update -yq \
&& apt-get install -yq --no-install-recommends \
$buildDeps \
python3-setuptools \
python3 \
python3-pip \
python3-requests \
apt-utils \
curl \
netcat \
locales \
&& sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
&& locale-gen \
&& update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
&& useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow
RUN python3 -m pip install -U pip \
&& pip3 install Cython \
&& pip3 install pytz \
&& pip3 install pyOpenSSL \
&& pip3 install ndg-httpsclient \
&& pip3 install pyasn1 \
&& pip3 install apache-airflow[crypto,postgres,s3,docker,log]==$AIRFLOW_VERSION \
&& pip3 install mesos.cli \
&& pip3 install boto3 \
&& pip3 install flake8 \
&& pip3 install psycopg2-binary \
&& pip3 install pytest \
&& apt-get remove --purge -yq $buildDeps \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/* \
/usr/share/man \
/usr/share/doc \
/usr/share/doc-base
# Add airflow user to root group to properly use the docker.sock
RUN gpasswd -a airflow root
WORKDIR ${AIRFLOW_HOME}
# Install dependencies
COPY requirements.txt .
RUN pip install -r requirements.txt
# Copy over dags, configuration code, and the entrypoint
COPY dags/ dags/
COPY scripts/entrypoint.sh .
COPY config/airflow.cfg .
# Copy over custom logger for s3 logs
COPY config/log_config.py config/log_config.py
COPY config/__init__.py config/__init__.py
# Make all files in ${AIRFLOW_HOME} owned by the airflow user
RUN chown -R airflow: ${AIRFLOW_HOME}
# Adds logging confir on the PYTHONPATH
ENV PYTHONPATH ${PYTHONPATH}:/usr/lib/python3.5/site-packages/:${AIRFLOW_HOME}/config/
EXPOSE 8080
USER airflow
ENTRYPOINT ["./entrypoint.sh"]
# This Dockerfile builds Mesos from source, making the python eggs available
# which is necessary for registering airflow as a framework with Mesos
FROM ubuntu:16.04
# Never prompts the user for choices on installation/configuration of packages
ENV DEBIAN_FRONTEND noninteractive
ENV TERM linux
# Mesos
ARG MESOS_VERSION=1.4.0
ARG MESOS_HOME=/usr/local/mesos
# Install Mesos Dependencies
RUN apt-get update -q --fix-missing
RUN apt-get install -y tar wget git
RUN apt-get install -y openjdk-8-jdk
RUN apt-get -y install build-essential python-dev python-six python-virtualenv libcurl4-nss-dev libsasl2-dev libsasl2-modules maven libapr1-dev libsvn-dev zlib1g-dev
# Install Mesos jar
RUN mkdir -p ${MESOS_HOME}
RUN wget -P ${MESOS_HOME} http://www.apache.org/dist/mesos/${MESOS_VERSION}/mesos-${MESOS_VERSION}.tar.gz
RUN cd ${MESOS_HOME} && tar -zxvf mesos-${MESOS_VERSION}.tar.gz
RUN cd ${MESOS_HOME}/mesos-${MESOS_VERSION} && ./configure && make -j4 V=0 install
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment