Skip to content

Instantly share code, notes, and snippets.

Szymon Przedwojski sprzedwojski

View GitHub Profile
View test_mapreduce_dag.py
from airflow.contrib.operators import dataproc_operator
from airflow.utils.trigger_rule import TriggerRule
from airflow.utils import dates
from airflow import models
from airflow.operators import bash_operator
import datetime
PARAMS = {
"user.name": "TODO",
"nameNode": "hdfs://",
View mapreduce.py
from airflow.contrib.operators import dataproc_operator
from airflow.utils import dates
from airflow import models
import datetime
from airflow.operators import bash_operator
from airflow.utils.trigger_rule import TriggerRule
PARAMS = {
"user.name": "szymon",
"nameNode": "hdfs://localhost:8020",
@sprzedwojski
sprzedwojski / decorators.py
Created Mar 14, 2019
A 15min primer on decorators in Python
View decorators.py
# DECORATORS
# https://realpython.com/primer-on-python-decorators/
# Calling higher-order functions.
# A function that takes another function and extends its behaviour (without modifying it).
import functools
def my_decorator(func):
@functools.wraps(func)
View example_gcp_vision.py
product_set_id = 'my-product-set-1'
product_set_create = CloudVisionProductSetCreateOperator(
product_set_id=product_set_id
# ...
)
for x in range(0, 1000):
product_create = CloudVisionProductCreateOperator(
product_id='my-product-{}'.format(x),
You can’t perform that action at this time.