A Dockerized dbt Workflow

1_A Dockerized dbt Workflow

# A Dockerized dbt Workflow

docker-compose.yaml

version: '3.8'

services:

  dbt-server:
    build:
      context: .
      dockerfile: Dockerfile
    command: [ "dbt-rpc", "serve" ]
    env_file: .env
    environment:
      # For compatibility with Dagster logs
      # https://github.com/dbt-labs/dbt-rpc/issues/63
      - DBT_ENABLE_LEGACY_LOGGER=True
    ports:
      - '8581:8580'
    volumes:
      - ./:/app
    networks:
      - default

  dbt-console:
    build:
      context: .
      dockerfile: Dockerfile
    command: [ "/bin/bash" ]
    env_file: .env
    stdin_open: true
    tty: true
    volumes:
      - ./:/app
    networks:
      - default

networks:
  default:
    driver: bridge

Dockerfile

FROM python:3.8.1-slim-buster

LABEL maintainer="Juan Gesino"

# Set working directory
WORKDIR /app

# Install OS dependencies
RUN apt-get update && apt-get install -qq -y \
    git gcc build-essential libpq-dev --fix-missing --no-install-recommends \ 
    && apt-get clean

# Make sure we are using latest pip
RUN pip install --upgrade pip

# Create directory for dbt config
RUN mkdir -p /root/.dbt

# Copy requirements.txt
COPY requirements.txt requirements.txt

# Install dependencies
RUN pip install -r requirements.txt

# Copy dbt profile
COPY profiles.yml /root/.dbt/profiles.yml

# Copy source code
COPY . /app

# Start the dbt RPC server
CMD ["dbt-rpc", "serve"]

profiles-pg.yml

config:
  send_anonymous_usage_stats: False
  use_colors: True

jaffle_shop:
  target: postgres
  outputs:
    postgres:
      type: postgres
      host: "{{ env_var('DB_HOST') }}"
      user: "{{ env_var('DB_USER') }}"
      pass: "{{ env_var('DB_PASSWORD') }}"
      port: "{{ env_var('DB_PORT') | int }}"
      dbname: "{{ env_var('DB_NAME') }}"
      schema: public
      threads: 4

profiles-sf.yml

config:
  send_anonymous_usage_stats: False
  use_colors: True

jaffle_shop:
  target: snowflake
  outputs:
    snowflake:
      type: snowflake
      account: "{{ env_var('SF_ACCOUNT') }}"
      user: "{{ env_var('SF_USER') }}"
      password: "{{ env_var('SF_PASSWORD') }}"
      role: "{{ env_var('SF_ROLE') }}"
      warehouse: "{{ env_var('SF_WAREHOUSE') }}"
      database: "{{ env_var('SF_DATABASE') }}"
      schema: "{{ env_var('SF_SCHEMA') }}"
      threads: 8