Skip to content

Instantly share code, notes, and snippets.

😁

Szymon Przedwojski sprzedwojski

😁
Block or report user

Report or block sprzedwojski

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View subdag_test.py
from airflow.utils import dates
from airflow.contrib.operators import dataproc_operator
from airflow import models
from airflow.utils.trigger_rule import TriggerRule
from o2a_libs.el_basic_functions import *
from airflow.operators import bash_operator
import datetime
from o2a_libs.el_wf_functions import *
from airflow.operators import dummy_operator
View test_mapreduce_dag.py
from airflow.contrib.operators import dataproc_operator
from airflow.utils.trigger_rule import TriggerRule
from airflow.utils import dates
from airflow import models
from airflow.operators import bash_operator
import datetime
PARAMS = {
"user.name": "TODO",
"nameNode": "hdfs://",
View mapreduce.py
from airflow.contrib.operators import dataproc_operator
from airflow.utils import dates
from airflow import models
import datetime
from airflow.operators import bash_operator
from airflow.utils.trigger_rule import TriggerRule
PARAMS = {
"user.name": "szymon",
"nameNode": "hdfs://localhost:8020",
@sprzedwojski
sprzedwojski / decorators.py
Created Mar 14, 2019
A 15min primer on decorators in Python
View decorators.py
# DECORATORS
# https://realpython.com/primer-on-python-decorators/
# Calling higher-order functions.
# A function that takes another function and extends its behaviour (without modifying it).
import functools
def my_decorator(func):
@functools.wraps(func)
View example_gcp_vision.py
product_set_id = 'my-product-set-1'
product_set_create = CloudVisionProductSetCreateOperator(
product_set_id=product_set_id
# ...
)
for x in range(0, 1000):
product_create = CloudVisionProductCreateOperator(
product_id='my-product-{}'.format(x),
You can’t perform that action at this time.