Skip to content

Instantly share code, notes, and snippets.

View kzzzr's full-sized avatar

Artemiy Kzr kzzzr

59 followers · 8 following
View GitHub Profile
@kzzzr
kzzzr / 1
Created August 13, 2019 20:57
1
@kzzzr
kzzzr / dbt.sh
Last active April 9, 2022 12:45
DE - DWH - Data Build Tool
# install dbt: https://docs.getdbt.com/dbt-cli/installation
pip install dbt==0.19.0
# clone repo
git clone https://github.com/dbt-labs/jaffle_shop
# create ~/.dbt/profiles.yml
# ensure profile setup
dbt debug
@kzzzr
kzzzr / DE - Dataproc - Hive.sh
Last active May 26, 2021 10:32
## Инициализация
# устанавливаем проект
gcloud projects list
gcloud config set project <my-project>
# выбираем зону для кластера, например, europe-west3-a
gcloud compute regions list
gcloud compute zones list
@kzzzr
kzzzr / file_formats.sh
Last active March 16, 2021 19:11
DE - File formats
## Инициализация
# устанавливаем проект
gcloud projects list
gcloud config set project my-project
# выбираем зону для кластера, например, europe-west3-a
gcloud compute regions list
gcloud compute zones list
@kzzzr
kzzzr / kiva.sql
Created August 25, 2020 08:50
HIVE - Kiva loans homework
drop table kiva_loans ;
create external table kiva_loans
(
id INT
, funded_amount FLOAT
, loan_amount FLOAT
, activity STRING
, sector STRING
, use STRING
@kzzzr
kzzzr / constraints-3.7.txt
Created November 8, 2020 21:00
# Editable install with no version control (apache-airflow==1.10.11)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
Flask-Babel==1.0.0
Flask-Bcrypt==0.7.1
Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
@kzzzr
kzzzr / log4j.properties
Created November 27, 2020 10:59
Spark-sql demo on GCP
log4j.rootLogger=ERROR, console
# set the log level for these components
log4j.logger.com.test=DEBUG
log4j.logger.org=ERROR
log4j.logger.org.apache.spark=ERROR
log4j.logger.org.spark-project=ERROR
log4j.logger.org.apache.hadoop=ERROR
log4j.logger.io.netty=ERROR
log4j.logger.org.apache.zookeeper=ERROR
@kzzzr
kzzzr / docker_airflow.sh
Created December 10, 2020 20:22
Docker Airflow Tutorial
# Домашнее задание: поднять Airflow в Docker, протестировать и запустить DAG tutorial.py
# link to docker-airflow repository
https://github.com/puckel/docker-airflow
# build docker image
docker build -t learn/docker .
## local execution
docker run -d \
@kzzzr
kzzzr / atlas_cli.sh
Last active March 31, 2022 17:38
Data Engineer – MongoDB practice
# define your connection string (URI) with env variable
# export MONGODB_ATLAS_URI=<your_uri>
mongo $MONGODB_ATLAS_URI
# replica set
rs.status()
db.isMaster()
# Mongo Shell Basics
@kzzzr
kzzzr / constraints-3.7.txt
Last active May 18, 2021 09:50
# Editable install with no version control (apache-airflow==1.10.15)
Babel==2.9.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
Flask-Babel==1.0.0
Flask-Bcrypt==0.7.1
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.0
Flask-Login==0.4.1
Flask-OpenID==1.2.5