Setup Airflow with DBT and other airflow plugins

setup_airflow_w_dbt.sh

#!/bin/bash

# Define custom ports
WEB_SERVER_PORT=8081
FLOWER_PORT=5556

# Create necessary directories
mkdir -p ./dags ./logs ./plugins

# Set environment variable for Airflow user
echo -e "AIRFLOW_UID=$(id -u)" > .env

# Download the docker-compose.yaml file
curl -LfO 'https://airflow.apache.org/docs/apache-airflow/2.10.2/docker-compose.yaml'

# Change permissions to make the file writable
chmod u+w docker-compose.yaml

# Modify the docker-compose.yaml file to use custom ports
sed -i "s/8080:8080/${WEB_SERVER_PORT}:8080/g" docker-compose.yaml
sed -i "s/5555:5555/${FLOWER_PORT}:5555/g" docker-compose.yaml

# Uncomment the build line and remove the image line
sed -i '/^\s*image:.*apache\/airflow/d' docker-compose.yaml
sed -i 's/^\s*# build: \./  build: ./' docker-compose.yaml

# Verify the modification
if grep -q '^\s*build: \.$' docker-compose.yaml; then
    echo "Successfully updated docker-compose.yaml with build context."
    sleep 5
else
    echo "Failed to update docker-compose.yaml with build context." >&2
    exit 1
fi
# Create a Dockerfile to install dbt and plugins
cat <<EOF > Dockerfile
FROM apache/airflow:2.10.2

# Install dbt
RUN pip install dbt-core dbt-postgres

# Install Airflow plugins
RUN pip install apache-airflow-providers-slack
RUN pip install apache-airflow-providers-amazon
RUN pip install apache-airflow-providers-google
RUN pip install apache-airflow-providers-postgres

# Switch to the airflow user
USER airflow
EOF

# Build the custom Docker image
docker-compose build

# Initialize the Airflow database
docker-compose up airflow-init

# Start Airflow services in detached mode
docker-compose up -d

# Output the access information
echo "Airflow is running on http://localhost:${WEB_SERVER_PORT}"
echo "Flower is running on http://localhost:${FLOWER_PORT}"