Created
April 7, 2019 19:51
-
-
Save jacbeekers/668ab84f1f4660be52bb778d1ecbfa5f to your computer and use it in GitHub Desktop.
Apache Airflow from scratch on RedHat Linux 6.5 base install
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Installed: Oracle Linux 6.5 Base system | |
===================== | |
Login as user root (or run the commands as user that has the privileges to create users) | |
===================== | |
1. As root, create the airflow user: | |
# useradd airflow | |
# passwd airflow | |
<type in a nice password><twice> | |
2. create the admins group | |
# groupadd admins | |
3. add user airflow to the admins group | |
# usermod -aG admins airflow | |
4. run visudo to allow the admins group to run the sudo command | |
# visudo | |
%admins ALL=(ALL) NOPASSWD: ALL | |
===================== | |
Login as user airflow | |
===================== | |
## | |
# Prereqs | |
## Download git-2.21.0.tar.gz and Python-3.6.8.tgz | |
sudo yum install gcc gcc-c++ openssl | |
sudo yum install curl curl-devel zlib zlib-devel | |
sudo yum install cyrus-sasl-devel.x86_64 | |
# prereq if airflow should use sqlite. This may already be installed on your system, but it wasn't on mine. | |
sudo yum install sqlite-devel | |
# prereq if airflow should use mysql (or if you're going to use airflow[all]) | |
sudo yum install mysql mysql-devel | |
# prereq if airflow should use postgresql (or if you're going to use airflow[all]) | |
## On RedHat 6.x the max version included is 8.4. Python3 requires 9.1 as minimum. so get a later version: | |
##yum install https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-6-x86_64/pgdg-redhat11-11-2.noarch.rpm | |
sudo yum install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-6-x86_64/pgdg-redhat96-9.6-3.noarch.rpm | |
sudo yum install postgresql96-devel | |
## remove the 8.4 version | |
sudo yum remove postgresql | |
sudo yum remove postgresql-devel | |
# For python and pip to use https: | |
sudo yum install openssl-devel | |
## | |
## some needed for mysql, some for python36/37, some for git. Next two unknown why these were initially installed. Not needed. | |
#sudo yum -y install @development | |
#sudo yum -y install libffi-devel | |
## | |
# Git | |
mkdir /appl/git | |
cd /home/airflow/Downloads/git | |
tar -zxvf git-2.21.0.tar.gz | |
cd git-2.21.0 | |
./configure --prefix=/appl/git --with-gitconfig=/appl/git/gitconfig | |
make | |
make install | |
# | |
# we've added a later version of postgresql than the one provided by RedHat 6.x, include it in PATH | |
PATH=/appl/git/bin:/appl/python36/bin:/usr/pgsql-9.6/bin:$PATH | |
export PATH | |
## | |
# Python | |
mkdir /appl/python36 | |
cd /home/airflow/Downloads/python/ | |
tar -zxvf Python-3.6.8.tgz | |
cd Python-3.6.8 | |
./configure --prefix=/appl/python36 --with-ensurepip=install | |
make | |
make install | |
## | |
# airflow | |
cd /appl | |
python3 -m venv /appl/airflow | |
source airflow/bin/activate | |
cd airflow/ | |
export SLUGIFY_USES_TEXT_UNIDECODE=yes | |
pip3 install apache-airflow[all] | |
#(downloading and installing many files) | |
#(running many, many, many setup.py's. The cassandra-driver takes a very long time (4-5 minutes). Be patient! | |
#optional: | |
#pip3 install --upgrade pip | |
## | |
# change all /home/airflow references in airflow.cfg to /appl/airflow, and ensure you have /appl/airflow and NOT /appl/airflow/airflow in $HOME/airflow/airflow.cfg | |
# in addition: load_examples = False | |
# | |
# ONLY THEN: | |
airflow initdb | |
## | |
nohup airflow webserver & | |
nohup airflow scheduler & |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment