Airflow installation in Linode instance

airflow_installation_linode.sh

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# 
# This software consists of voluntary contributions made by many individuals
# and is licensed under the MIT license. For more information, see
# <http://www.doctrine-project.org>.

# Set of instructions to deploy airflow using pip in a linode instance
# @license http://www.opensource.org/licenses/mit-license.html  MIT License
# @author Andrea Montes <mamontesp@gmail.com>
 
## 1. Create a ssh connection to linode instance 
## 1.1 Create a ssh key in your local 
## 1.2 include in your ~/.ssh/config file the next four lines. Replace the text inside curly brackets {}
### Host {host_alias}
### Hostname {ip}
### IdentityFile ~/.ssh/{mykey}
### User {youruser}
## 1.3 Copy that ssh key to your linode instance. 
ssh-copy-id -i ~/.ssh/mykey user@host
## it will prompt you for password

## ssh to your linode 
ssh host_alias
## 2. create a folder to locate your airflow dags
mkdir airflow
cd airflow
## 3. Create a virtual environment
python3 -m venv env
## 3.1 If you get an error, make sure you install pythonenv
sudo apt install python3.10-venv
## 4. Enable your virtual environment
source env/bin/activate
## 5. Install airflow with pip
## 5.1 Without any other dependencies
pip install apache-airflow
## 5.2 With celery
pip install "apache-airflow[celery]==2.5.1" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-3.7.txt"
## 6. Init the development db to track dags and other airflow metadata
airflow db init
## 7. Verify you have access to a list of dags (created for testing purposes)
airflow dags list
## 8. Create a admin user
airflow users  create --role Admin --username admin --email admin --firstname admin --lastname admin --password admin
## 9. Start airflow webserver
airflow webserver
## 10. Go to your web browser and navigate to your linode ip address, port 8080
100.00.00.00:8080
## 11. Login with your admin user
## 12. Enjoy!
 
 
 
## How to create a daemon to enable access to Airflow
## Create a sh script describing the actions required to start the airflow webserver

## 1. Create a sh script in /usr/local/bin describing the actions required to start the airflow webserver. 
## For this example the name of the script is run_airflow_server.sh

#!/bin/bash
cd /home/username/Documents/airflow
source env/bin/activate
airflow webserver

## 2. Add exec permissions to the file
chmod +x run_airflow_server.sh

## 3. Navigate to the folder /etc/systemd/system 

## 4. Create a file airflow.service with this content
[Unit]
Description=This is the runner for airflow with pip from python

[Service]
Type=simple
ExecStart=/usr/local/bin/run_airflow_webserver.sh

[Install]
WantedBy=multi-user.target

## 5. Run these commands

sudo chmod 664 airflow.service #giving permissions
sudo systemctl daemon-reload   #after editing or creating any file in /etc/systemd/system/ the files have to be reloaded
sudo systemct enable --now airflow.service # starting the service, with --now you start and enable service immediately.
sudo systemctl status airflow.service # see status of your service

## You should be able to navigate in your webbrowser to your ip address and see airflow running. You can close your ssh connection and still be able to access it.