Run Airbyte generated DBT files on Huawei Cloud DWS

dws-airbyte-dbt.sh

#!/usr/bin/env bash
WORKSPACE_ID=$1
NORMALIZE_DIR="$(pwd)/dbtnormalization"

if [ -z "$WORKSPACE_ID" ]; then
  echo "Workspace ID is required"
  exit 1
fi

echo "Installing dbt"
pip install dbt-postgres==1.5.4 > /dev/null

echo "Creating dbt dependency folder"
mkdir /dbt

echo "Cleaning old dbt files"
rm -rf $NORMALIZE_DIR/normalization-files
mkdir -p $NORMALIZE_DIR/normalization-files
cd $NORMALIZE_DIR

echo "Copying dbt normalization files"
docker cp airbyte-server:/tmp/workspace/$WORKSPACE_ID/normalize/ $NORMALIZE_DIR/normalization-files
echo "Processing dbt normalization files"
find ./ -type f -name "*.sql" | while read -r sql_file; do
  echo "Processing $sql_file"
  temp_file="${sql_file}_temp"
  awk '!/indexes/' $sql_file >$temp_file
  mv $temp_file $sql_file
done

cd $NORMALIZE_DIR/normalization-files/normalize
echo "Installing dbt dependencies"
dbt deps

echo "Running dbt"
dbt run