cordon-thiago

version: '2'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"
  kafka:
    image: wurstmeister/kafka:2.11-1.1.1
    ports:
      - "9092:9092"

cordon-thiago

    spark-worker-n:
        image: bitnami/spark:3.0.1
        networks:
            - default_net
        environment:
            - SPARK_MODE=worker
            - SPARK_MASTER_URL=spark://spark:7077
            - SPARK_WORKER_MEMORY=1G
            - SPARK_WORKER_CORES=1
            - SPARK_RPC_AUTHENTICATION_ENABLED=no

cordon-thiago

import sys
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession

##########################
# You can configure master here if you do not pass the spark.master paramenter in conf
##########################
#master = "spark://spark:7077"
#conf = SparkConf().setAppName("Spark Hello World").setMaster(master)
#sc = SparkContext(conf=conf)

cordon-thiago

from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator
from datetime import datetime, timedelta

###############################################
# Parameters
###############################################
spark_master = "spark://spark:7077"

cordon-thiago

cordon-thiago

cordon-thiago

cordon-thiago

cordon-thiago

cordon-thiago

def oversampleSMOTE(X, y):
    '''
    Resample a dataset using SMOTE oversample

    Input:
        X = dataframe with x variables (explanatory variables)
        y = dataframe with y variable (variable to predict)

    Output:
        df[0] = X dataframe resampled