dprateek1991

# Airflow DEMO DAG

from airflow import DAG
from datetime import timedelta, datetime
from kubernetes.client import models as k8s
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator

args = {
	"owner": "prateek.dubey",
	"email": ["dataengineeringe2e@gmail.com"],

dprateek1991

# Import Spark Library

from pyspark.sql import SparkSession

# Create a Spark Context

spark = SparkSession.builder \
        .appName('amazon-data-review') \
        .config("spark.kubernetes.driver.master", "k8s://https://14HH948AC611F5A7F020B62A5C366F04.yl4.us-east-1.eks.amazonaws.com:443") \
        .config("spark.kubernetes.namespace", "spark") \

dprateek1991

# Import Spark Library

from pyspark.sql import SparkSession

# Create a Spark Context

spark = SparkSession.builder \
        .appName('amazon-data-review') \
        .getOrCreate()

dprateek1991

apiVersion: v1
kind: Pod
metadata:
  name: spark-jump-pod
  namespace: spark
spec:
  serviceAccountName: spark
  containers:
  -  image: dataengineeringe2e/spark-ubuntu-3.0.1
     name: spark-jump-pod

dprateek1991

kubectl create namespace spark
kubectl create serviceaccount spark -n spark
kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=spark:spark --namespace=spark

dprateek1991

/opt/spark/bin/spark-submit \
        --master k8s://https://rancher.example.com:6443 \
        --deploy-mode cluster \
        --name amazon-data-review \
        --conf spark.kubernetes.driver.pod.name=amazon-data-review \
        --conf spark.kubernetes.executor.podNamePrefix=amazon-data-review \
        --conf spark.kubernetes.namespace=spark \
        --conf spark.executor.instances=2 \
        --conf spark.executor.cores=3 \
        --conf spark.executor.memory=55g \

dprateek1991

/opt/spark/bin/spark-submit \
        --master k8s://https://14HH948AC611F5A7F020B62A5C366F04.yl4.us-east-1.eks.amazonaws.com:443 \
        --deploy-mode cluster \
        --name amazon-data-review \
        --conf spark.kubernetes.driver.pod.name=amazon-data-review \
        --conf spark.kubernetes.executor.podNamePrefix=amazon-data-review \
        --conf spark.kubernetes.namespace=spark \
        --conf spark.executor.instances=2 \
        --conf spark.executor.cores=3 \
        --conf spark.executor.memory=55g \

dprateek1991

# Airflow DEMO DAG
from airflow import DAG
from datetime import timedelta, datetime
from kubernetes.client import models as k8s
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
 
args = {
   "owner": "prateek.dubey",
   "email": ["dataengineeringe2e@gmail.com"],
   "depends_on_past": False,

dprateek1991

apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
  name: amazon-data-review
  namespace: spark
spec:
  type: Python
  pythonVersion: "3"
  mode: cluster
  image: "dataengineeringe2e/spark-ubuntu-3.0.1"