Running dbt using Gitlab CI/CD

.gitlab-ci.dbt_run.yml

.dbt_run: &dbt_jobs
  image: python:3.8.1-slim-buster
  before_script:
    - apt-get update
    - apt-get dist-upgrade -y
    - apt-get install -y --no-install-recommends git make ca-certificates libpq-dev libicu-dev pkg-config gcc build-essential
    - pip install dbt-postgres==1.0.0
    - export CI_PROFILE_TARGET="--profiles-dir . --target ${TARGET_NAME}"
    - echo $CI_PROFILE_TARGET

.gitlab-ci.deploy.yml

.deploy:
  <<: *dbt_jobs
  script:
    - dbt deps $CI_PROFILE_TARGET
    - dbt build $CI_PROFILE_TARGET

.gitlab-ci.stage.yml

deploy-production:
  extends: .deploy
  stage: deploy-production
  variables:
    TARGET_NAME: postgres
  when: manual

.gitlab-ci.yml

stages:
  - deploy-production

.dbt_run: &dbt_jobs
  image: python:3.8.1-slim-buster
  before_script:
    - apt-get update
    - apt-get dist-upgrade -y
    - apt-get install -y --no-install-recommends git make ca-certificates libpq-dev libicu-dev pkg-config gcc build-essential
    - pip install dbt-postgres==1.0.0
    - export CI_PROFILE_TARGET="--profiles-dir . --target ${TARGET_NAME}"
    - echo $CI_PROFILE_TARGET

.deploy:
  <<: *dbt_jobs
  script:
    - dbt deps $CI_PROFILE_TARGET
    - dbt build $CI_PROFILE_TARGET

deploy-production:
  extends: .deploy
  stage: deploy-production
  variables:
    TARGET_NAME: postgres
  when: manual

1_Running dbt using Gitlab CICD

# Running dbt using Gitlab CI/CD

profiles.yml

config:
  send_anonymous_usage_stats: False
  use_colors: True

jaffle_shop:
  target: postgres
  outputs:
    postgres:
      type: postgres
      host: "{{ env_var('DB_HOST') }}"
      user: "{{ env_var('DB_USER') }}"
      pass: "{{ env_var('DB_PASSWORD') }}"
      port: "{{ env_var('DB_PORT') | int }}"
      dbname: "{{ env_var('DB_NAME') }}"
      schema: public
      threads: 4

trigger_pipeline.py

import requests

GITLAB_PIPELINE_DBT = "XXXXXXXXXXXXXXXX"
PROJECT_ID = "XXXXXXXXX"
PIPELINE_ENDPOINT = f"https://gitlab.com/api/v4/projects/{PROJECT_ID}/trigger/pipeline"

BRANCH_NAME = "main"


payload = {
    "token": GITLAB_PIPELINE_DBT,
    "ref": branch,
    "variables[EXAMPLE_VARIABLE]": "your_value_here",
}

response = requests.request("POST", PIPELINE_ENDPOINT, data=payload)